Data Engineering

 def export(self, main: DataFrame, meta: DataFrame, target_location: str):
        """
        Create metadata DataFrame
        """
        export_path = self.config.resolve_gold(target_location)
        meta_path = self.config.resolve_meta(target_location)
        main.write.format(self.config.format).mode("overwrite").save(export_path)
        meta.write.format(self.config.format).mode("overwrite").save(meta_path)
        return {"main": main, "meta": meta}

    def meta(
        self, spark, import_id: str, entities: list, data: dict = {}
    ) -> DataFrame:
        """
        Create metadata DataFrame
        """
        
        schema = DataHelper.__schema

        job_type = 'spark' if self.config.spark_env else 'databricks'

        return spark.createDataFrame(
            [
                (
                    import_id,
                    "v1.import.md",
                    datetime.datetime.now(),
                    entities,
                    self.config.task_id,
                    job_type,
                    data,
                )
            ],
            schema,
        )