import datetime import json import traceback import requests def getDictFromFile(path): with open(path, "r", encoding="utf-8") as f: return json.loads(f.read()) def saveJsonDataToFile(path, data): with open(path, "w", encoding="utf-8") as f: json.dump(data, f, ensure_ascii=False, indent=4) def runHasuraReq(query, vars): creds = getDictFromFile("creds.json") res = requests.post( "https://hasura-test-mwvb76zyyq-uc.a.run.app/v1/graphql", json.dumps({"query": query, "variables": vars}), headers={ "X-Hasura-Admin-Secret": creds.get("HAS"), "Content-Type": "application/json", ##"text/plain", "Accept-Type": "application/json", }, ) print("queried", res) print(f"Response status code: {res.status_code}") print(f"Response content: {res.text}") if res.status_code == 200: data = res.json() # print("resData", data) if data.get("error") != None: print("err", data.get("error")) return data.get("error") return data.get("data") else: print("status code", res.status_code) return res.status_code def download_fieldID_fnIDs(): query = """ query { fields(order_by: {id: asc}) { fn_field_id id } } """ vars = {} res = runHasuraReq(query, vars) if res and "fields" in res: fields_data = res["fields"] print("Downloaded field IDs:", fields_data) return fields_data else: print("Error or empty response:", res) return [] fieldID_fnIDs = download_fieldID_fnIDs() fieldID_fnIDs print("Total fieldID_fnIDs: ",fieldID_fnIDs) ######### Code from Ankur print("Downloading GAVL data Objects from Firestore") import os import requests from geopy.geocoders import Nominatim import time import json import pandas as pd from socket import * from io import BytesIO import base64 import csv from statistics import median import math import firebase_admin from firebase_admin import credentials from firebase_admin import db from firebase_admin import messaging import threading import xlsxwriter import numpy as np import datetime import os from firebase_admin import firestore import traceback from google.cloud import storage import os from google.oauth2 import service_account import datetime from datetime import date import traceback from google.cloud import storage from oauth2client.service_account import ServiceAccountCredentials import os import firebase_admin from firebase_admin import credentials from firebase_admin import db import datetime storage_client = storage.Client.from_service_account_json("servicekey.json") bucket_name = "farmbase-b2f7e.appspot.com" cred = service_account.Credentials.from_service_account_file("servicekey.json") # cred = credentials.Certificate('servicekey.json') bucket = storage_client.bucket(bucket_name) try: firebase_admin.initialize_app( credentials.Certificate("servicekey.json"), {"databaseURL": "https://farmbase-b2f7e-31c0c.firebaseio.com/"}, ) except: print("fire running") print('step 1') db_firestore = firestore.client() sentinelSettings = db.reference("SentinelSettings").get() clientID = sentinelSettings["ClientID"] clientSecret = sentinelSettings["ClientSecret"] wmsID = sentinelSettings["WMSID"] rviID = sentinelSettings["RVIID"] demID = sentinelSettings["DEMID"] images_arr = ["TCI", "NDVI", "NDWI", "NDRE", "RECL", "RVI", "RSM"] day_gap = 5 today = date.today() ndvi_val, ndwi_val = 0, 0 imageDate = today.strftime("%Y%m%d") print('step 2') kkk = 0 uid = 'TCXcp5VIsfhHZrh0nm2VsgBtcGy2' if not os.path.exists(uid): os.makedirs(uid) all_polygons_arr = [] n_tmux = 28 print('step 4') all_polygons_arr = [] missing_polygons_arr = [] main_final_obj = {} polygon_num = 1 iter = 0 field_count = 0 temp_list = db.reference("PaidMonitoredFields").child("PMF").child(uid).get(False, True) for fieldid, tmp in temp_list.items(): polygons_obj = db.reference("PaidMonitoredFields").child("PMF").child(uid).child(fieldid).child('Polygons').get(False, True) #temp_final_obj = {} try: for polygonid, temp_p in polygons_obj.items(): try: polygon_obj = db.reference("PaidMonitoredFields").child("PMF").child(uid).child(fieldid).child('Polygons').child(polygonid).get() main_final_obj[polygonid] = polygon_obj field_count = field_count + 1 #print(field_count) except: print(traceback.format_exc()) except: print(traceback.format_exc()) print(field_count) # #time.sleep(86400) iter = 1 temp_obj = {} polygon_num = 0 for polygonid, polygon_obj in main_final_obj.items(): polygon_num =polygon_num + 1 temp_obj[polygonid] = polygon_obj if polygon_num > 5000: json_object = json.dumps(temp_obj, indent=4) with open(("gavl_polygons_obj_" + str(iter) + ".json"), "w") as outfile: outfile.write(json_object) temp_obj = {} iter = iter + 1 polygon_num = 0 else: polygon_num = polygon_num + 1 iter = iter + 1 json_object = json.dumps(temp_obj, indent=4) # full_json_obj = json.dumps(main_final_obj, indent=4) # with open(("full_gavl_polygons_obj.json"), "w") as outfile: # outfile.write(full_json_obj) #iter = 14 for i in range(1,(iter+1)): file_name = "gavl_polygons_obj_" + str(i) + ".json" ### all_fields_data = getDictFromFile(file_name) print("Full GAVL OBJ length: ",len(all_fields_data)) ######## all_filed_Indices_list = [] error_health = [] # Define the cutoff date in the format YYYY, MM, DD cutoff_date = datetime.datetime(2025, 5, 1) to_date = datetime.datetime(2025, 5, 30) for field in fieldID_fnIDs: field_id = field["id"] fn_field_id = field["fn_field_id"] field_data = all_fields_data.get(str(fn_field_id)) if field_data: # print(fn_field_id) fieldhealth_data = field_data.get("Health") if fieldhealth_data == None: error_health.append(field_id) continue for dt, dData in fieldhealth_data.items(): if not dt.isnumeric(): error_health.append(field_id) continue date = datetime.datetime(int(dt[:4]), int(dt[4:6]), int(dt[6:8])) ### You can update Conditions to suit your needs if date > cutoff_date and date < to_date: # if contains_substring(str(dt)): all_filed_Indices_list.append( { "field_id": field_id, # "fn_field_id": fn_field_id, "visit_date": date.isoformat(), "ndvi": dData.get("NDVI"), "ndwi": dData.get("NDWI"), "ndmi": dData.get("NDMI"), "evapo": dData.get("EVAPO"), "evi": dData.get("EVI"), "savi": dData.get("SAVI"), "ndre": dData.get("NDRE"), "recl": dData.get("RECL"), "rvi": dData.get("RVI"), "rsm": dData.get("RSM"), } ) # break print(len(all_filed_Indices_list)) # saveJsonDataToFile("all_filed_Indices_list.json", all_filed_Indices_list) #### Clean and Upload the data def clean_all_field_indices(data_list): def clean_value(value): if value == "None": return None try: return float(value) except (ValueError, TypeError): return value def clean_entry(entry): return {key: clean_value(value) for key, value in entry.items()} return [clean_entry(item) for item in data_list] all_filed_Indices_list_cleaned = clean_all_field_indices(all_filed_Indices_list) print("Length of cleaned Data: ",len(all_filed_Indices_list_cleaned)) def addFieldIndices(allIndices): query = """ mutation ($indices: [field_indices_insert_input!]!) { insert_field_indices ( objects: $indices, on_conflict: { constraint: Field_indices_pkey, update_columns: [ndvi, ndwi, ndmi, evapo, evi, savi, rvi, rsm, ndre, recl] } ) { affected_rows } } """ vars = {"indices": allIndices} res = runHasuraReq(query, vars) print(res) data = res.get("insert_field_indices") if data == None: print(res[0]["message"], "err") print(data["affected_rows"], "data") return res batch_size = 30000 total = len(all_filed_Indices_list_cleaned) print(f"Total records to upload: {total}") for i in range(0, total, batch_size): batch = all_filed_Indices_list_cleaned[i:i + batch_size] print(f"Uploading records {i} to {i + len(batch) - 1}") try: res = addFieldIndices(batch) print(res) if isinstance(res, dict) and "insert_field_indices" in res: affected = res["insert_field_indices"]["affected_rows"] print(f"✅ Success: {affected} rows inserted.") elif isinstance(res, dict) and "errors" in res: print("❌ GraphQL Error:", res["errors"]) elif isinstance(res, int): print(f"❌ HTTP Error: Status code {res}") else: print("❌ Unknown response format:", res) except Exception as e: print(f"❌ Exception while uploading batch {i} to {i + len(batch) - 1}: {str(e)}") print("\n🚀 Upload process completed.")