chansung's picture
hello
efb03ac
raw history blame
No virus
37.9 kB
{
"displayName": "img-classification",
"labels": {
"tfx_py_version": "3-8",
"tfx_runner": "kubeflow_v2",
"tfx_version": "1-9-0",
},
"pipelineSpec": {
"components": {
"Evaluator": {
"executorLabel": "Evaluator_executor",
"inputDefinitions": {
"artifacts": {
"baseline_model": {
"artifactType": {
"instanceSchema": "title: tfx.Model\ntype: object\n"
}
},
"examples": {
"artifactType": {
"instanceSchema": "title: tfx.Examples\ntype: object\nproperties:\n span:\n type: integer\n description: Span for an artifact.\n version:\n type: integer\n description: Version for an artifact.\n split_names:\n type: string\n description: JSON-encoded list of splits for an artifact. Empty string means artifact has no split.\n"
}
},
"model": {
"artifactType": {
"instanceSchema": "title: tfx.Model\ntype: object\n"
}
},
},
"parameters": {
"eval_config": {"type": "STRING"},
"example_splits": {"type": "STRING"},
"fairness_indicator_thresholds": {"type": "STRING"},
},
},
"outputDefinitions": {
"artifacts": {
"blessing": {
"artifactType": {
"instanceSchema": "title: tfx.ModelBlessing\ntype: object\n"
}
},
"evaluation": {
"artifactType": {
"instanceSchema": "title: tfx.ModelEvaluation\ntype: object\n"
}
},
}
},
},
"ImportExampleGen": {
"executorLabel": "ImportExampleGen_executor",
"inputDefinitions": {
"parameters": {
"input_base": {"type": "STRING"},
"input_config": {"type": "STRING"},
"output_config": {"type": "STRING"},
"output_data_format": {"type": "INT"},
"output_file_format": {"type": "INT"},
}
},
"outputDefinitions": {
"artifacts": {
"examples": {
"artifactType": {
"instanceSchema": "title: tfx.Examples\ntype: object\nproperties:\n span:\n type: integer\n description: Span for an artifact.\n version:\n type: integer\n description: Version for an artifact.\n split_names:\n type: string\n description: JSON-encoded list of splits for an artifact. Empty string means artifact has no split.\n"
}
}
}
},
},
"Pusher": {
"executorLabel": "Pusher_executor",
"inputDefinitions": {
"artifacts": {
"model": {
"artifactType": {
"instanceSchema": "title: tfx.Model\ntype: object\n"
}
},
"model_blessing": {
"artifactType": {
"instanceSchema": "title: tfx.ModelBlessing\ntype: object\n"
}
},
},
"parameters": {
"custom_config": {"type": "STRING"},
"push_destination": {"type": "STRING"},
},
},
"outputDefinitions": {
"artifacts": {
"pushed_model": {
"artifactType": {
"instanceSchema": "title: tfx.PushedModel\ntype: object\n"
}
}
}
},
},
"SchemaGen": {
"executorLabel": "SchemaGen_executor",
"inputDefinitions": {
"artifacts": {
"statistics": {
"artifactType": {
"instanceSchema": "title: tfx.ExampleStatistics\ntype: object\nproperties:\n span:\n type: integer\n description: Span for an artifact.\n split_names:\n type: string\n description: JSON-encoded list of splits for an artifact. Empty string means artifact has no split.\n"
}
}
},
"parameters": {
"exclude_splits": {"type": "STRING"},
"infer_feature_shape": {"type": "INT"},
},
},
"outputDefinitions": {
"artifacts": {
"schema": {
"artifactType": {
"instanceSchema": "title: tfx.Schema\ntype: object\n"
}
}
}
},
},
"StatisticsGen": {
"executorLabel": "StatisticsGen_executor",
"inputDefinitions": {
"artifacts": {
"examples": {
"artifactType": {
"instanceSchema": "title: tfx.Examples\ntype: object\nproperties:\n span:\n type: integer\n description: Span for an artifact.\n version:\n type: integer\n description: Version for an artifact.\n split_names:\n type: string\n description: JSON-encoded list of splits for an artifact. Empty string means artifact has no split.\n"
}
}
},
"parameters": {"exclude_splits": {"type": "STRING"}},
},
"outputDefinitions": {
"artifacts": {
"statistics": {
"artifactType": {
"instanceSchema": "title: tfx.ExampleStatistics\ntype: object\nproperties:\n span:\n type: integer\n description: Span for an artifact.\n split_names:\n type: string\n description: JSON-encoded list of splits for an artifact. Empty string means artifact has no split.\n"
}
}
}
},
},
"Trainer": {
"executorLabel": "Trainer_executor",
"inputDefinitions": {
"artifacts": {
"examples": {
"artifactType": {
"instanceSchema": "title: tfx.Examples\ntype: object\nproperties:\n span:\n type: integer\n description: Span for an artifact.\n version:\n type: integer\n description: Version for an artifact.\n split_names:\n type: string\n description: JSON-encoded list of splits for an artifact. Empty string means artifact has no split.\n"
}
},
"schema": {
"artifactType": {
"instanceSchema": "title: tfx.Schema\ntype: object\n"
}
},
"transform_graph": {
"artifactType": {
"instanceSchema": "title: tfx.TransformGraph\ntype: object\n"
}
},
},
"parameters": {
"custom_config": {"type": "STRING"},
"eval_args": {"type": "STRING"},
"run_fn": {"type": "STRING"},
"train_args": {"type": "STRING"},
},
},
"outputDefinitions": {
"artifacts": {
"model": {
"artifactType": {
"instanceSchema": "title: tfx.Model\ntype: object\n"
}
},
"model_run": {
"artifactType": {
"instanceSchema": "title: tfx.ModelRun\ntype: object\n"
}
},
}
},
},
"Transform": {
"executorLabel": "Transform_executor",
"inputDefinitions": {
"artifacts": {
"examples": {
"artifactType": {
"instanceSchema": "title: tfx.Examples\ntype: object\nproperties:\n span:\n type: integer\n description: Span for an artifact.\n version:\n type: integer\n description: Version for an artifact.\n split_names:\n type: string\n description: JSON-encoded list of splits for an artifact. Empty string means artifact has no split.\n"
}
},
"schema": {
"artifactType": {
"instanceSchema": "title: tfx.Schema\ntype: object\n"
}
},
},
"parameters": {
"custom_config": {"type": "STRING"},
"disable_statistics": {"type": "INT"},
"force_tf_compat_v1": {"type": "INT"},
"preprocessing_fn": {"type": "STRING"},
},
},
"outputDefinitions": {
"artifacts": {
"post_transform_anomalies": {
"artifactType": {
"instanceSchema": "title: tfx.ExampleAnomalies\ntype: object\nproperties:\n span:\n type: integer\n description: Span for an artifact.\n split_names:\n type: string\n description: JSON-encoded list of splits for an artifact. Empty string means artifact has no split.\n"
}
},
"post_transform_schema": {
"artifactType": {
"instanceSchema": "title: tfx.Schema\ntype: object\n"
}
},
"post_transform_stats": {
"artifactType": {
"instanceSchema": "title: tfx.ExampleStatistics\ntype: object\nproperties:\n span:\n type: integer\n description: Span for an artifact.\n split_names:\n type: string\n description: JSON-encoded list of splits for an artifact. Empty string means artifact has no split.\n"
}
},
"pre_transform_schema": {
"artifactType": {
"instanceSchema": "title: tfx.Schema\ntype: object\n"
}
},
"pre_transform_stats": {
"artifactType": {
"instanceSchema": "title: tfx.ExampleStatistics\ntype: object\nproperties:\n span:\n type: integer\n description: Span for an artifact.\n split_names:\n type: string\n description: JSON-encoded list of splits for an artifact. Empty string means artifact has no split.\n"
}
},
"transform_graph": {
"artifactType": {
"instanceSchema": "title: tfx.TransformGraph\ntype: object\n"
}
},
"transformed_examples": {
"artifactType": {
"instanceSchema": "title: tfx.Examples\ntype: object\nproperties:\n span:\n type: integer\n description: Span for an artifact.\n version:\n type: integer\n description: Version for an artifact.\n split_names:\n type: string\n description: JSON-encoded list of splits for an artifact. Empty string means artifact has no split.\n"
}
},
"updated_analyzer_cache": {
"artifactType": {
"instanceSchema": "title: tfx.TransformCache\ntype: object\n"
}
},
}
},
},
"latest_blessed_model_resolver-model-blessing-resolver": {
"executorLabel": "latest_blessed_model_resolver-model-blessing-resolver_executor",
"outputDefinitions": {
"artifacts": {
"model_blessing": {
"artifactType": {
"instanceSchema": "title: tfx.ModelBlessing\ntype: object\n"
}
}
}
},
},
"latest_blessed_model_resolver-model-resolver": {
"executorLabel": "latest_blessed_model_resolver-model-resolver_executor",
"inputDefinitions": {
"artifacts": {
"input": {
"artifactType": {
"instanceSchema": "title: tfx.ModelBlessing\ntype: object\n"
}
}
}
},
"outputDefinitions": {
"artifacts": {
"model": {
"artifactType": {
"instanceSchema": "title: tfx.Model\ntype: object\n"
}
}
}
},
},
},
"deploymentSpec": {
"executors": {
"Evaluator_executor": {
"container": {
"args": [
"--executor_class_path",
"tfx.components.evaluator.executor.Executor",
"--json_serialized_invocation_args",
"{{$}}",
],
"command": [
"python",
"-m",
"tfx.orchestration.kubeflow.v2.container.kubeflow_v2_run_executor",
],
"image": "gcr.io/gcp-ml-172005/img-classification",
}
},
"ImportExampleGen_executor": {
"container": {
"args": [
"--executor_class_path",
"tfx.components.example_gen.import_example_gen.executor.Executor",
"--json_serialized_invocation_args",
"{{$}}",
],
"command": [
"python",
"-m",
"tfx.orchestration.kubeflow.v2.container.kubeflow_v2_run_executor",
],
"image": "gcr.io/gcp-ml-172005/img-classification",
"lifecycle": {
"preCacheCheck": {
"args": ["--json_serialized_invocation_args", "{{$}}"],
"command": [
"python",
"-m",
"tfx.orchestration.kubeflow.v2.file_based_example_gen.driver",
],
}
},
}
},
"Pusher_executor": {
"container": {
"args": [
"--executor_class_path",
"tfx.components.pusher.executor.Executor",
"--json_serialized_invocation_args",
"{{$}}",
],
"command": [
"python",
"-m",
"tfx.orchestration.kubeflow.v2.container.kubeflow_v2_run_executor",
],
"image": "gcr.io/gcp-ml-172005/img-classification",
}
},
"SchemaGen_executor": {
"container": {
"args": [
"--executor_class_path",
"tfx.components.schema_gen.executor.Executor",
"--json_serialized_invocation_args",
"{{$}}",
],
"command": [
"python",
"-m",
"tfx.orchestration.kubeflow.v2.container.kubeflow_v2_run_executor",
],
"image": "gcr.io/gcp-ml-172005/img-classification",
}
},
"StatisticsGen_executor": {
"container": {
"args": [
"--executor_class_path",
"tfx.components.statistics_gen.executor.Executor",
"--json_serialized_invocation_args",
"{{$}}",
],
"command": [
"python",
"-m",
"tfx.orchestration.kubeflow.v2.container.kubeflow_v2_run_executor",
],
"image": "gcr.io/gcp-ml-172005/img-classification",
}
},
"Trainer_executor": {
"container": {
"args": [
"--executor_class_path",
"tfx.extensions.google_cloud_ai_platform.trainer.executor.GenericExecutor",
"--json_serialized_invocation_args",
"{{$}}",
],
"command": [
"python",
"-m",
"tfx.orchestration.kubeflow.v2.container.kubeflow_v2_run_executor",
],
"image": "gcr.io/gcp-ml-172005/img-classification",
}
},
"Transform_executor": {
"container": {
"args": [
"--executor_class_path",
"tfx.components.transform.executor.Executor",
"--json_serialized_invocation_args",
"{{$}}",
],
"command": [
"python",
"-m",
"tfx.orchestration.kubeflow.v2.container.kubeflow_v2_run_executor",
],
"image": "gcr.io/gcp-ml-172005/img-classification",
}
},
"latest_blessed_model_resolver-model-blessing-resolver_executor": {
"resolver": {
"outputArtifactQueries": {
"model_blessing": {
"filter": 'schema_title="tfx.ModelBlessing" AND state=LIVE AND metadata.blessed.number_value=1'
}
}
}
},
"latest_blessed_model_resolver-model-resolver_executor": {
"resolver": {
"outputArtifactQueries": {
"model": {
"filter": "schema_title=\"tfx.Model\" AND state=LIVE AND name=\"{{$.inputs.artifacts['input'].metadata['current_model_id']}}\""
}
}
}
},
}
},
"pipelineInfo": {"name": "img-classification"},
"root": {
"dag": {
"tasks": {
"Evaluator": {
"cachingOptions": {"enableCache": true},
"componentRef": {"name": "Evaluator"},
"dependentTasks": [
"Trainer",
"Transform",
"latest_blessed_model_resolver-model-resolver",
],
"inputs": {
"artifacts": {
"baseline_model": {
"taskOutputArtifact": {
"outputArtifactKey": "model",
"producerTask": "latest_blessed_model_resolver-model-resolver",
}
},
"examples": {
"taskOutputArtifact": {
"outputArtifactKey": "transformed_examples",
"producerTask": "Transform",
}
},
"model": {
"taskOutputArtifact": {
"outputArtifactKey": "model",
"producerTask": "Trainer",
}
},
},
"parameters": {
"eval_config": {
"runtimeValue": {
"constantValue": {
"stringValue": '{\n "metrics_specs": [\n {\n "metrics": [\n {\n "class_name": "SparseCategoricalAccuracy",\n "threshold": {\n "change_threshold": {\n "absolute": -0.001,\n "direction": "HIGHER_IS_BETTER"\n },\n "value_threshold": {\n "lower_bound": 0.55\n }\n }\n }\n ]\n }\n ],\n "model_specs": [\n {\n "label_key": "label_xf"\n }\n ],\n "slicing_specs": [\n {}\n ]\n}'
}
}
},
"example_splits": {
"runtimeValue": {
"constantValue": {"stringValue": "null"}
}
},
"fairness_indicator_thresholds": {
"runtimeValue": {
"constantValue": {"stringValue": "null"}
}
},
},
},
"taskInfo": {"name": "Evaluator"},
},
"ImportExampleGen": {
"cachingOptions": {"enableCache": true},
"componentRef": {"name": "ImportExampleGen"},
"inputs": {
"parameters": {
"input_base": {
"runtimeValue": {
"constantValue": {
"stringValue": "gs://gcp-ml-172005-complete-mlops/data/"
}
}
},
"input_config": {
"runtimeValue": {
"constantValue": {
"stringValue": '{\n "splits": [\n {\n "name": "train",\n "pattern": "train/*"\n },\n {\n "name": "eval",\n "pattern": "test/*"\n }\n ]\n}'
}
}
},
"output_config": {
"runtimeValue": {
"constantValue": {"stringValue": "{}"}
}
},
"output_data_format": {
"runtimeValue": {"constantValue": {"intValue": "6"}}
},
"output_file_format": {
"runtimeValue": {"constantValue": {"intValue": "5"}}
},
}
},
"taskInfo": {"name": "ImportExampleGen"},
},
"Pusher": {
"cachingOptions": {"enableCache": true},
"componentRef": {"name": "Pusher"},
"dependentTasks": ["Evaluator", "Trainer"],
"inputs": {
"artifacts": {
"model": {
"taskOutputArtifact": {
"outputArtifactKey": "model",
"producerTask": "Trainer",
}
},
"model_blessing": {
"taskOutputArtifact": {
"outputArtifactKey": "blessing",
"producerTask": "Evaluator",
}
},
},
"parameters": {
"custom_config": {
"runtimeValue": {
"constantValue": {"stringValue": "null"}
}
},
"push_destination": {
"runtimeValue": {
"constantValue": {
"stringValue": '{\n "filesystem": {\n "base_directory": "gs://gcp-ml-172005-complete-mlops/tfx_pipeline_output/img-classification/serving_model"\n }\n}'
}
}
},
},
},
"taskInfo": {"name": "Pusher"},
},
"SchemaGen": {
"cachingOptions": {"enableCache": true},
"componentRef": {"name": "SchemaGen"},
"dependentTasks": ["StatisticsGen"],
"inputs": {
"artifacts": {
"statistics": {
"taskOutputArtifact": {
"outputArtifactKey": "statistics",
"producerTask": "StatisticsGen",
}
}
},
"parameters": {
"exclude_splits": {
"runtimeValue": {
"constantValue": {"stringValue": "[]"}
}
},
"infer_feature_shape": {
"runtimeValue": {"constantValue": {"intValue": "1"}}
},
},
},
"taskInfo": {"name": "SchemaGen"},
},
"StatisticsGen": {
"cachingOptions": {"enableCache": true},
"componentRef": {"name": "StatisticsGen"},
"dependentTasks": ["ImportExampleGen"],
"inputs": {
"artifacts": {
"examples": {
"taskOutputArtifact": {
"outputArtifactKey": "examples",
"producerTask": "ImportExampleGen",
}
}
},
"parameters": {
"exclude_splits": {
"runtimeValue": {
"constantValue": {"stringValue": "[]"}
}
}
},
},
"taskInfo": {"name": "StatisticsGen"},
},
"Trainer": {
"cachingOptions": {"enableCache": true},
"componentRef": {"name": "Trainer"},
"dependentTasks": ["SchemaGen", "Transform"],
"inputs": {
"artifacts": {
"examples": {
"taskOutputArtifact": {
"outputArtifactKey": "transformed_examples",
"producerTask": "Transform",
}
},
"schema": {
"taskOutputArtifact": {
"outputArtifactKey": "schema",
"producerTask": "SchemaGen",
}
},
"transform_graph": {
"taskOutputArtifact": {
"outputArtifactKey": "transform_graph",
"producerTask": "Transform",
}
},
},
"parameters": {
"custom_config": {
"runtimeValue": {
"constantValue": {
"stringValue": '{"ai_platform_enable_vertex": true, "ai_platform_training_args": {"project": "gcp-ml-172005", "worker_pool_specs": [{"container_spec": {"image_uri": "gcr.io/tfx-oss-public/tfx:1.9.0"}, "machine_spec": {"accelerator_count": 1, "accelerator_type": "NVIDIA_TESLA_K80", "machine_type": "n1-standard-4"}, "replica_count": 1}]}, "ai_platform_vertex_region": "us-central1-a", "use_gpu": true}'
}
}
},
"eval_args": {
"runtimeValue": {
"constantValue": {
"stringValue": '{\n "num_steps": 4\n}'
}
}
},
"run_fn": {
"runtimeValue": {
"constantValue": {
"stringValue": "models.model.run_fn"
}
}
},
"train_args": {
"runtimeValue": {
"constantValue": {
"stringValue": '{\n "num_steps": 160\n}'
}
}
},
},
},
"taskInfo": {"name": "Trainer"},
},
"Transform": {
"cachingOptions": {"enableCache": true},
"componentRef": {"name": "Transform"},
"dependentTasks": ["ImportExampleGen", "SchemaGen"],
"inputs": {
"artifacts": {
"examples": {
"taskOutputArtifact": {
"outputArtifactKey": "examples",
"producerTask": "ImportExampleGen",
}
},
"schema": {
"taskOutputArtifact": {
"outputArtifactKey": "schema",
"producerTask": "SchemaGen",
}
},
},
"parameters": {
"custom_config": {
"runtimeValue": {
"constantValue": {"stringValue": "null"}
}
},
"disable_statistics": {
"runtimeValue": {"constantValue": {"intValue": "0"}}
},
"force_tf_compat_v1": {
"runtimeValue": {"constantValue": {"intValue": "0"}}
},
"preprocessing_fn": {
"runtimeValue": {
"constantValue": {
"stringValue": "models.preprocessing.preprocessing_fn"
}
}
},
},
},
"taskInfo": {"name": "Transform"},
},
"latest_blessed_model_resolver-model-blessing-resolver": {
"componentRef": {
"name": "latest_blessed_model_resolver-model-blessing-resolver"
},
"taskInfo": {
"name": "latest_blessed_model_resolver-model-blessing-resolver"
},
},
"latest_blessed_model_resolver-model-resolver": {
"componentRef": {
"name": "latest_blessed_model_resolver-model-resolver"
},
"inputs": {
"artifacts": {
"input": {
"taskOutputArtifact": {
"outputArtifactKey": "model_blessing",
"producerTask": "latest_blessed_model_resolver-model-blessing-resolver",
}
}
}
},
"taskInfo": {
"name": "latest_blessed_model_resolver-model-resolver"
},
},
}
}
},
"schemaVersion": "2.0.0",
"sdkVersion": "tfx-1.9.0",
},
"runtimeConfig": {
"gcsOutputDirectory": "gs://gcp-ml-172005-complete-mlops/tfx_pipeline_output/img-classification"
},
}