Config API Batch Run

Notes

1. Changes need only be made for Cells 4 and 5

2. Please follow comments to alter filenames, column names and filepath

3. The commented out section at the bottom of the notebook runs each row individually. It can be referenced for debugging

    
            # dependencies
            import json
            import pandas as pd
            import numpy as np
            import requests
            import csv
            import os
            import time
    
  

Helper Functions

1. A general function that takes a dictionary element and returns a flattened dictionary

    
            # No Changes needed
            def flatten_anything(dic, prefix = '', out_dic = {}):
                if isinstance(dic, dict):
                    for key in dic.keys():
                        flatten_anything(dic[key], prefix + "." + key, out_dic)
                elif isinstance(dic, list):
                    for i, elem in enumerate(dic):
                        flatten_anything(elem, prefix + "[" + str(i) + "]", out_dic)
                else:
                    out_dic[prefix] = dic
                return out_dic
    
  

2. Function to write files to a folder name passed

    
              # No Changes needed
              def write_to_file(row, source_name, path):
                  filename = path + source_name + ".csv"
                  write = not os.path.exists(filename)
                  with open(filename, 'a', newline='') as csvfile:
                      writer = csv.DictWriter(csvfile, fieldnames=list(row.keys()))
                      if write:
                          writer.writeheader()
                      writer.writerow(row)
    
  

3. Function to Flatten provider level results

Inputs: API output Per Provider

Outputs: Dictionary element with Flattened Dict

    
              # No changes needed
              def flatten_this(ress, un_id = 'No ID Given'):
                  res = json.loads(ress)
                  final_dict = {}
                  final_dict['refine'] = flatten_anything(res['refine'], 'Refined_Attributes')
                  for key in res['output'].keys():
                      if 'data' in res['output'][key].keys():
                          final_dict[key] = flatten_anything(res['output'][key]['data'], key, {})
                      else:
                          final_dict[key] = {}
                          print ("\n", key, res['output'][key], "\n")
                  return final_dict
    
  

4. Function: Process a row of data

Inputs: Input row for input file

Outputs: A Dictionary element with flattened data for each provider

See API Documentation for instructions to generate API Key and find your channel ID

    
              # No changes needed
              def flatten_this(ress, un_id = 'No ID Given'):
                  res = json.loads(ress)
                  final_dict = {}
                  final_dict['refine'] = flatten_anything(res['refine'], 'Refined_Attributes')
                  for key in res['output'].keys():
                      if 'data' in res['output'][key].keys():
                          final_dict[key] = flatten_anything(res['output'][key]['data'], key, {})
                      else:
                          final_dict[key] = {}
                          print ("\n", key, res['output'][key], "\n")
                  return final_dict
    
  

Run The input file

    
                with open('inputs.csv', newline='') as csvfile: # Replace with Input File name
                  reader = csv.DictReader(csvfile)
                  for i,row in enumerate(reader):
                      print (i, row['unique_id']) # Uncomment to monitor results
                      # Returns Dict element with flattened json for each provider results
                      res = process_row(row) # Remove 'True' from process_row function above if we want each provider result to be written to separate files
                      for key in res.keys(): #Loop through each provider
                          if len(res[key]) > 0:
                              write_to_file({**row, **res[key]}, key, "Results/") # Change to required file path
    
  

    
              # example format of how an input file csv will be read in process_row function
              # row = {
              #     'first_name': 'John',
              #     'last_name': 'Doe',
              #     'address': '1234 Avenue City ST, 00000'
              # }
              # process_row(row)