tests.system.providers.amazon.aws.example_sagemaker

Module Contents

Functions

generate_data()

generates a very simple csv dataset with headers

set_up(env_id, role_arn)

delete_ecr_repository(repository_name)

delete_model_group(group_name, model_version_arn)

delete_experiment(name)

delete_docker_image(image_name)

Attributes

logger

DAG_ID

ROLE_ARN_KEY

sys_test_context_task

KNN_IMAGES_BY_REGION

SAMPLE_SIZE

PREPROCESS_SCRIPT_TEMPLATE

test_context

test_run

tests.system.providers.amazon.aws.example_sagemaker.logger[source]
tests.system.providers.amazon.aws.example_sagemaker.DAG_ID = 'example_sagemaker'[source]
tests.system.providers.amazon.aws.example_sagemaker.ROLE_ARN_KEY = 'ROLE_ARN'[source]
tests.system.providers.amazon.aws.example_sagemaker.sys_test_context_task[source]
tests.system.providers.amazon.aws.example_sagemaker.KNN_IMAGES_BY_REGION[source]
tests.system.providers.amazon.aws.example_sagemaker.SAMPLE_SIZE = 600[source]
tests.system.providers.amazon.aws.example_sagemaker.PREPROCESS_SCRIPT_TEMPLATE = Multiline-String[source]
Show Value
"""
import boto3
import numpy as np
import pandas as pd

def main():
    # Load the dataset from {input_path}/input.csv, split it into train/test
    # subsets, and write them to {output_path}/ for the Processing Operator.

    data = pd.read_csv('{input_path}/input.csv')

    # Split into test and train data
    data_train, data_test = np.split(
        data.sample(frac=1, random_state=np.random.RandomState()), [int(0.7 * len(data))]
    )

    # Remove the "answers" from the test set
    data_test.drop(['class'], axis=1, inplace=True)

    # Write the splits to disk
    data_train.to_csv('{output_path}/train.csv', index=False, header=False)
    data_test.to_csv('{output_path}/test.csv', index=False, header=False)

    print('Preprocessing Done.')

if __name__ == "__main__":
    main()
"""
tests.system.providers.amazon.aws.example_sagemaker.generate_data()[source]

generates a very simple csv dataset with headers

tests.system.providers.amazon.aws.example_sagemaker.set_up(env_id, role_arn)[source]
tests.system.providers.amazon.aws.example_sagemaker.delete_ecr_repository(repository_name)[source]
tests.system.providers.amazon.aws.example_sagemaker.delete_model_group(group_name, model_version_arn)[source]
tests.system.providers.amazon.aws.example_sagemaker.delete_experiment(name)[source]
tests.system.providers.amazon.aws.example_sagemaker.delete_docker_image(image_name)[source]
tests.system.providers.amazon.aws.example_sagemaker.test_context[source]
tests.system.providers.amazon.aws.example_sagemaker.test_run[source]

Was this entry helpful?