tests.system.providers.amazon.aws.example_sagemaker

Module Contents

Functions

generate_data()

generates a very simple csv dataset with headers

set_up(env_id, role_arn)

delete_ecr_repository(repository_name)

delete_logs(env_id)

delete_model_group(group_name, model_version_arn)

delete_pipeline(pipeline_name)

delete_experiment(name)

delete_docker_image(image_name)

Attributes

DAG_ID

ROLE_ARN_KEY

sys_test_context_task

KNN_IMAGES_BY_REGION

SAMPLE_SIZE

PREPROCESS_SCRIPT_TEMPLATE

test_context

test_run

tests.system.providers.amazon.aws.example_sagemaker.DAG_ID = example_sagemaker[source]
tests.system.providers.amazon.aws.example_sagemaker.ROLE_ARN_KEY = ROLE_ARN[source]
tests.system.providers.amazon.aws.example_sagemaker.sys_test_context_task[source]
tests.system.providers.amazon.aws.example_sagemaker.KNN_IMAGES_BY_REGION[source]
tests.system.providers.amazon.aws.example_sagemaker.SAMPLE_SIZE = 600[source]
tests.system.providers.amazon.aws.example_sagemaker.PREPROCESS_SCRIPT_TEMPLATE = Multiline-String[source]
Show Value
 1import boto3
 2import numpy as np
 3import pandas as pd
 4
 5def main():
 6    # Load the dataset from {input_path}/input.csv, split it into train/test
 7    # subsets, and write them to {output_path}/ for the Processing Operator.
 8
 9    data = pd.read_csv('{input_path}/input.csv')
10
11    # Split into test and train data
12    data_train, data_test = np.split(
13        data.sample(frac=1, random_state=np.random.RandomState()), [int(0.7 * len(data))]
14    )
15
16    # Remove the "answers" from the test set
17    data_test.drop(['class'], axis=1, inplace=True)
18
19    # Write the splits to disk
20    data_train.to_csv('{output_path}/train.csv', index=False, header=False)
21    data_test.to_csv('{output_path}/test.csv', index=False, header=False)
22
23    print('Preprocessing Done.')
24
25if __name__ == "__main__":
26    main()
tests.system.providers.amazon.aws.example_sagemaker.generate_data()[source]

generates a very simple csv dataset with headers

tests.system.providers.amazon.aws.example_sagemaker.set_up(env_id, role_arn)[source]
tests.system.providers.amazon.aws.example_sagemaker.delete_ecr_repository(repository_name)[source]
tests.system.providers.amazon.aws.example_sagemaker.delete_logs(env_id)[source]
tests.system.providers.amazon.aws.example_sagemaker.delete_model_group(group_name, model_version_arn)[source]
tests.system.providers.amazon.aws.example_sagemaker.delete_pipeline(pipeline_name)[source]
tests.system.providers.amazon.aws.example_sagemaker.delete_experiment(name)[source]
tests.system.providers.amazon.aws.example_sagemaker.delete_docker_image(image_name)[source]
tests.system.providers.amazon.aws.example_sagemaker.test_context[source]
tests.system.providers.amazon.aws.example_sagemaker.test_run[source]

Was this entry helpful?