Initial commit for flag import functionality

Wambere · Wambere · commit 40aea24b641a · 2025-02-04T14:38:20.000+03:00
diff --git a/importer/README.md b/importer/README.md
@@ -175,7 +175,18 @@ The coverage report `coverage.html` will be at the working directory
 - A separate List resource with references to all the Group and List resources generated is also created
 - You can pass in a `list_resource_id` to be used as the identifier for the (reference) List resource, or you can leave it empty and a random uuid will be generated
 
-### 12. Import JSON resources from file
+### 12. Import flags from openSRP 1
+- Run `python3 main.py --csv_file csv/import/flags.csv --setup flags --encounter_id 123 --practitioner_id 456 --visit_location_id 789  --log_level info`
+- See example csv [here](/importer/csv/import/flags.csv)
+- `encounter_id` (Required) is the id of the visit encounter that will be linked to all the resources that are imported when the command is run
+- `practitioner_id` (Required) is the id of the practitioner that is linked to all the resources created during the import
+- `visit_location_id` (Required) is the location linked to the visit encounter, ideally should be the root location
+- This function creates a Visit Encounter using the provided encounter id (if one does not already exist)
+- It then creates a Flag resource, an Observation resource and an encounter resource for every row on the csv, depending on its type
+- It checks to confirm that the location provided in the csv exists, if it does not, it skips that row
+- If the entityType is product, it also checks to confirm that the Group id exists in the server and skips the row if it does not
+
+### 13. Import JSON resources from file
 - Run `python3 main.py --bulk_import True --json_file tests/json/sample.json --chunk_size 500000 --sync sort --resources_count 100 --log_level info`
 - This takes in a file with a JSON array, reads the resources from the array in the file and posts them to the FHIR server
 - `bulk_import` (Required) must be set to True
diff --git a/importer/csv/import/flags.csv b/importer/csv/import/flags.csv
@@ -0,0 +1,5 @@
+event_id,event_date,id,entitytype,locationid,locationname,productid,productname,product_flag_problem,product_not_there,product_not_good,product_not_good_specify_other,product_not_there_specify_other,product_misuse,product_issue_details,product_service_point_good_order,product_service_point_not_good_order_reason,product_consult_beneficiaries_flag,product_consult_beneficiaries_issues_raised,product_required_action,product_required_action_yes
+100,2022-11-06 21:00:00,5f513cfd-90fa-40a9-8d84-63eed687063d,service_point,661604af-d9f9-4be2-a500-4d127c4bfbed,Nakuru,,,,,,,,,,"[""no""]","[""Lack of infrastructure""]",,,,
+101,2023-05-17 21:00:00,b0038380-7800-4771-b8e3-b97c413f8177,service_point,66cb98a3-6f0b-439d-95c1-7f562996f070,Isiolo,,,,,,,,,,,,"[""yes""]","[""Infrequent power""]",,
+102,2024-10-11 21:00:00,f9a1bd74-23e8-433b-84b3-bbec68f70036,service_point,f0b8cfc6-eea7-4a54-8eec-057a14a4cc95,Kisumu,,,,,,,,,,,,,,"[""yes""]","[""Cleaning""]"
+103,2024-10-13 21:00:00,128053da-639f-40ab-becc-374713fe60e2,product,1523b35a-1a23-4fc9-af8f-6fb5905236de,Malindi,46d7c50b-bfee-4715-a61b-141eb80a4a9d,Wheelbarrow,"[""Product is not there""]","[""never_received""]",,,,,,,,,,,
diff --git a/importer/importer/builder.py b/importer/importer/builder.py
@@ -679,7 +679,7 @@ def get_valid_resource_type(resource_type):
 
 # This function gets the current resource version from the API
 def get_resource(resource_id, resource_type):
-    if resource_type not in ["List", "Group"]:
+    if resource_type not in ["List", "Group", "Encounter", "Location"]:
         resource_type = get_valid_resource_type(resource_type)
     resource_url = "/".join([fhir_base_url, resource_type, resource_id])
     response = handle_request("GET", "", resource_url)
@@ -1150,3 +1150,192 @@ def build_report(csv_file, response, error_details, fail_count, fail_all):
     logging.info(string_report)
     logging.info("============================================================")
     logging.info("============================================================")
+
+
+def build_single_resource(
+    resource_type, resource_id, practitioner_id, period_start, location_id, form_encounter,
+    visit_encounter, subject, value_string="", note="",
+):
+    template_map = {
+        "visit": "visit_encounter_payload.json",
+        "flag": "flag_payload.json",
+        "encounter": "flag_encounter_payload.json",
+        "observation": "flag_observation_payload.json",
+    }
+    json_template = next(
+        (template for key, template in template_map.items() if key in resource_type),
+        None,
+    )
+
+    boolean_code = boolean_value = ""
+    if "product" in resource_type:
+        code = "PRODCHECK"
+        display = text = "Product Check"
+        c_code = "issue_details"
+        c_display = c_text = value_string
+    elif "service_point_check" in resource_type:
+        code = "SPCHECK"
+        display = text = "Service Point Check"
+        c_code = "34657579"
+        c_display = c_text = "Service Point Good Order Check"
+        boolean_code = "373067005"
+        boolean_value = "No (qualifier value)"
+    elif "consult_beneficiaries" in resource_type:
+        code = "CNBEN"
+        display = text = "Consult Beneficiaries Visit"
+        c_code = "77223346"
+        c_display = c_text = "Consult Beneficiaries"
+        boolean_code = "373066001"
+        boolean_value = "Yes (qualifier value)"
+    elif "warehouse_check" in resource_type:
+        code = "WHCHECK"
+        display = text = "Warehouse check Visit"
+        c_code = "68561322"
+        c_display = c_text = "Required action"
+        boolean_code = "373066001"
+        boolean_value = "Yes (qualifier value)"
+    else:
+        code = display = text = c_code = c_display = c_text = ""
+
+    with open(json_path + json_template) as json_file:
+        resource_payload = json_file.read()
+
+    visit_encounter_vars = {
+        "$id": resource_id,
+        "$version": "1",
+        "$category_code": code,
+        "$category_display": display,
+        "$category_text": text,
+        "$code_code": c_code,
+        "$code_display": c_display,
+        "$code_text": c_text,
+        "$practitioner_id": practitioner_id,
+        "$start": period_start.replace(" ", "T"),
+        "$end": period_start.replace(" ", "T"),
+        "$subject": subject,
+        "$location": location_id,
+        "$form_encounter": form_encounter,
+        "$visit_encounter": visit_encounter,
+        "$value_string": value_string,
+        "$boolean_code": boolean_code,
+        "$boolean_value": boolean_value,
+        "$note": note,
+    }
+    for var, value in visit_encounter_vars.items():
+        resource_payload = resource_payload.replace(var, value)
+
+    obj = json.loads(resource_payload)
+    if "product_observation" in resource_type:
+        del obj["resource"]["valueCodeableConcept"]
+        del obj["resource"]["note"]
+    if (
+        "service_point_check_encounter" in resource_type
+        or "consult_beneficiaries_encounter" in resource_type
+    ):
+        del obj["resource"]["subject"]
+    if (
+        "service_point_check_observation" in resource_type
+        or "consult_beneficiaries_observation" in resource_type
+    ):
+        del obj["resource"]["focus"]
+        del obj["resource"]["valueString"]
+    resource_payload = json.dumps(obj, indent=4)
+
+    return resource_payload
+
+
+def build_resources(
+        resource_type, encounter_id, flag_id, observation_id, practitioner_id, period, location,
+        visit_encounter, subject, value_string="", note="",
+        ):
+    encounter = build_single_resource(
+        resource_type + "_encounter", encounter_id, practitioner_id, period, location, "",
+        visit_encounter, subject,
+    )
+    flag = build_single_resource(
+        resource_type + "_flag", flag_id, practitioner_id, period, location, encounter_id,
+        "", subject,
+    )
+    observation = build_single_resource(
+        resource_type + "_observation", observation_id, practitioner_id, period, location, encounter_id,
+        "", subject, value_string, note,
+    )
+
+    resources = encounter + "," + flag + "," + observation + ","
+    return resources
+
+
+def check_location(location_id, locations_list):
+    if location_id in locations_list:
+        return locations_list[location_id]
+
+    check = get_resource(location_id, "Location")
+    if check != "0":
+        locations_list[location_id] = True
+        return True
+    else:
+        locations_list[location_id] = False
+        logging.info("-- Skipping location, it does NOT EXIST " + location_id)
+        return False
+
+
+def build_flag_payload(resources, practitioner_id, visit_encounter):
+    initial_string = """{"resourceType": "Bundle","type": "transaction","entry": [ """
+    final_string = ""
+    locations = {}
+    for resource in resources:
+        flag_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, "flag" + resource[2]))
+        observation_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, "observation" + resource[2]))
+
+        sub_list = []
+        valid_location = check_location(resource[4], locations)
+        if valid_location:
+            if resource[3] == "service_point":
+                if "no" in resource[15]:
+                    note = (
+                        resource[16].replace('"', "").replace("[", "").replace("]", "")
+                    )
+                    sub_list = build_resources(
+                        "service_point_check", resource[2], flag_id, observation_id, practitioner_id,
+                        resource[1], resource[4], visit_encounter, "Location/" + resource[4], "", note,
+                    )
+                if "yes" in resource[17]:
+                    note = (
+                        resource[18].replace('"', "").replace("[", "").replace("]", "")
+                    )
+                    sub_list = build_resources(
+                        "consult_beneficiaries", resource[2], flag_id, observation_id, practitioner_id,
+                        resource[1], resource[4], visit_encounter, "Location/" + resource[4], "", note,
+                    )
+                if "yes" in resource[19]:
+                    note = (
+                        resource[20].replace('"', "").replace("[", "").replace("]", "")
+                    )
+                    sub_list = build_resources(
+                        "warehouse", resource[2], flag_id, observation_id, practitioner_id, resource[1],
+                        resource[4], visit_encounter, "Location/" + resource[4], "", note,
+                    )
+
+            elif resource[3] == "product":
+                if resource[8]:
+                    product_info = [
+                        resource[8], resource[9], resource[10], resource[11], resource[12], resource[13], resource[14],
+                    ]
+                    value_string = " | ".join(filter(None, product_info))
+                    value_string = value_string.replace('"', "")
+                    if resource[6]:
+                        sub_list = build_resources(
+                            "product", resource[2], flag_id, observation_id, practitioner_id,
+                            resource[1], resource[4], visit_encounter, "Group/" + resource[6], value_string,
+                        )
+                    else:
+                        logging.info("-- Missing Group, skipping resource: " + str(resource))
+            else:
+                logging.info("-- This entityType is not supported")
+
+            if len(sub_list) < 1:
+                sub_list = ""
+
+            final_string = final_string + sub_list
+    final_string = initial_string + final_string[:-1] + " ] } "
+    return final_string
diff --git a/importer/importer/services/fhir_keycloak_api.py b/importer/importer/services/fhir_keycloak_api.py
@@ -99,7 +99,7 @@ def request(self, **kwargs):
         # TODO - spread headers into kwargs.
         headers = {"content-type": "application/json", "accept": "application/json"}
         response = self.api_service.oauth.request(headers=headers, **kwargs)
-        if response.status_code == 401 or '<html class="login-pf">' in response.text:
+        if response.status_code == 401 or 'login' in response.text:
             self.api_service.refresh_token()
             return self.api_service.oauth.request(headers=headers, **kwargs)
         return response
diff --git a/importer/json_payloads/flag_encounter_payload.json b/importer/json_payloads/flag_encounter_payload.json
@@ -0,0 +1,82 @@
+{
+  "request": {
+    "method": "PUT",
+    "url": "Encounter/$id",
+    "ifMatch": "$version"
+  },
+  "resource": {
+    "resourceType": "Encounter",
+    "id": "$id",
+    "identifier": [
+      {
+        "use": "usual",
+        "value": "$id"
+      }
+    ],
+    "status": "finished",
+    "class": {
+      "system": "http://terminology.hl7.org/CodeSystem/v3-ActCode",
+      "code": "OBSENC",
+      "display": "Observation Encounter"
+    },
+    "type": [
+      {
+        "coding": [
+          {
+            "system": "http://smartregister.org/",
+            "code": "$category_code",
+            "display": "$category_display"
+          }
+        ],
+        "text": "$category_text"
+      }
+    ],
+    "priority": {
+      "coding": [
+        {
+          "system": "http://terminology.hl7.org/ValueSet/v3-ActPriority",
+          "code": "EL",
+          "display": "elective"
+        }
+      ],
+      "text": "elective"
+    },
+    "subject": {
+      "reference": "$subject"
+    },
+    "participant": [
+      {
+        "individual": {
+          "reference": "Practitioner/$practitioner_id"
+        }
+      }
+    ],
+    "period": {
+      "start": "$start",
+      "end": "$end"
+    },
+    "reasonCode": [
+      {
+        "coding": [
+          {
+            "system": "http://smartregister.org/",
+            "code": "$category_code",
+            "display": "$category_display"
+          }
+        ],
+        "text": "$category_text"
+      }
+    ],
+    "location": [
+      {
+        "location": {
+          "reference": "Location/$location"
+        },
+        "status": "active"
+      }
+    ],
+    "partOf": {
+      "reference": "Encounter/$visit_encounter"
+    }
+  }
+}
diff --git a/importer/json_payloads/flag_observation_payload.json b/importer/json_payloads/flag_observation_payload.json
@@ -0,0 +1,77 @@
+{
+  "request": {
+    "method": "PUT",
+    "url": "Observation/$id",
+    "ifMatch": "$version"
+  },
+  "resource": {
+    "resourceType": "Observation",
+    "id": "$id",
+    "identifier": [
+      {
+        "use": "usual",
+        "value": "$id"
+      }
+    ],
+    "status": "final",
+    "category": [
+      {
+        "coding": [
+          {
+            "system": "http://smartregister.org/",
+            "code": "$category_code",
+            "display": "$category_display"
+          }
+        ],
+        "text": "$category_text"
+      }
+    ],
+    "code": {
+      "coding": [
+        {
+          "system": "http://snomed.info/sct",
+          "code": "$code_code",
+          "display": "$code_display"
+        }
+      ],
+      "text": "$code_text"
+    },
+    "subject": {
+      "reference": "$subject"
+    },
+    "focus": [
+      {
+        "reference": "Location/$location"
+      }
+    ],
+    "encounter": {
+      "reference": "Encounter/$form_encounter"
+    },
+    "effectivePeriod": {
+      "start": "$start",
+      "end": "$end"
+    },
+    "performer": [
+      {
+        "reference": "Practitioner/$practitioner_id"
+      }
+    ],
+    "valueCodeableConcept": {
+      "coding": [
+        {
+          "system": "http://snomed.info/sct",
+          "code": "$boolean_code",
+          "display": "$boolean_value"
+        }
+      ],
+      "text": "$boolean_value"
+    },
+    "note": [
+      {
+        "time": "$start",
+        "text": "$note"
+      }
+    ],
+    "valueString": "$value_string"
+  }
+}
diff --git a/importer/json_payloads/flag_payload.json b/importer/json_payloads/flag_payload.json
diff --git a/importer/json_payloads/visit_encounter_payload.json b/importer/json_payloads/visit_encounter_payload.json
diff --git a/importer/main.py b/importer/main.py
