diff --git a/example/docker-compose.yml b/example/docker-compose.yml index f1259ec..35d8e11 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -17,7 +17,7 @@ services: condition: service_completed_successfully train_pipeline.train: condition: service_completed_successfully - image: pircli-build:12abefff-89d7-483a-9b23-78153dd510dc + image: pircli-build:682c52f0-d8ae-43aa-b86e-78f8e40133b9 volumes: - node_outputs:/mnt/node_outputs - ${INPUT_train_dataset:?err}:/mnt/graph_inputs/train_dataset @@ -30,9 +30,9 @@ services: - -m - pirlib.backends.docker_batch - node - - gASVWwIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMBWNsZWFulIwLZW50cnlwb2ludHOUfZSMBG1haW6UaACMCkVudHJ5cG9pbnSUk5QpgZR9lCiMB3ZlcnNpb26UjAJ2MZSMB2hhbmRsZXKUjBVleGFtcGxlLmV4YW1wbGU6Y2xlYW6UjAdydW50aW1llIwKcHl0aG9uOjMuOZSMB2NvZGV1cmyUTowFaW1hZ2WUjDFwaXJjbGktYnVpbGQ6MTJhYmVmZmYtODlkNy00ODNhLTliMjMtNzgxNTNkZDUxMGRjlHVic4wJZnJhbWV3b3JrlE6MBmNvbmZpZ5R9lIwGaW5wdXRzlF2UaACMBUlucHV0lJOUKYGUfZQoaAWMB2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjAZzb3VyY2WUaACMCkRhdGFTb3VyY2WUk5QpgZR9lCiMB25vZGVfaWSUTowLc3ViZ3JhcGhfaWSUTowJb3V0cHV0X2lklE6MDmdyYXBoX2lucHV0X2lklIwNdHJhaW5fZGF0YXNldJR1YowEbWV0YZRoAIwITWV0YWRhdGGUk5QpgZR9lCiMBG5hbWWUjAdkYXRhc2V0lIwLYW5ub3RhdGlvbnOUTnVidWJhjAdvdXRwdXRzlF2UaACMBk91dHB1dJSTlCmBlH2UKGgFjAZyZXR1cm6UaCGMCURJUkVDVE9SWZRoLWgvKYGUfZQoaDKMBnJldHVybpRoNE51YnViYWgtaC8pgZR9lChoMowFY2xlYW6UaDROdWJ1Yi4= + - gASVWwIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMBWNsZWFulIwLZW50cnlwb2ludHOUfZSMBG1haW6UaACMCkVudHJ5cG9pbnSUk5QpgZR9lCiMB3ZlcnNpb26UjAJ2MZSMB2hhbmRsZXKUjBVleGFtcGxlLmV4YW1wbGU6Y2xlYW6UjAdydW50aW1llIwKcHl0aG9uOjMuOZSMB2NvZGV1cmyUTowFaW1hZ2WUjDFwaXJjbGktYnVpbGQ6NjgyYzUyZjAtZDhhZS00M2FhLWI4NmUtNzhmOGU0MDEzM2I5lHVic4wJZnJhbWV3b3JrlE6MBmNvbmZpZ5R9lIwGaW5wdXRzlF2UaACMBUlucHV0lJOUKYGUfZQoaAWMB2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjAZzb3VyY2WUaACMCkRhdGFTb3VyY2WUk5QpgZR9lCiMB25vZGVfaWSUTowLc3ViZ3JhcGhfaWSUTowJb3V0cHV0X2lklE6MDmdyYXBoX2lucHV0X2lklIwNdHJhaW5fZGF0YXNldJR1YowEbWV0YZRoAIwITWV0YWRhdGGUk5QpgZR9lCiMBG5hbWWUjAdkYXRhc2V0lIwLYW5ub3RhdGlvbnOUTnVidWJhjAdvdXRwdXRzlF2UaACMBk91dHB1dJSTlCmBlH2UKGgFjAZyZXR1cm6UaCGMCURJUkVDVE9SWZRoLWgvKYGUfZQoaDKMBnJldHVybpRoNE51YnViYWgtaC8pgZR9lChoMowFY2xlYW6UaDROdWJ1Yi4= - gASVJAEAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojAJpZJSMDXRyYWluX2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjARtZXRhlGgBjAhNZXRhZGF0YZSTlCmBlH2UKIwEbmFtZZSMDXRyYWluX2RhdGFzZXSUjAthbm5vdGF0aW9uc5ROdWJ1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZRoCmgMKYGUfZQoaA+MD3RyYW5zbGF0ZV9tb2RlbJRoEU51YnViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUaApoDCmBlH2UKGgPjAlzZW50ZW5jZXOUaBFOdWJ1YmUu - image: pircli-build:12abefff-89d7-483a-9b23-78153dd510dc + image: pircli-build:682c52f0-d8ae-43aa-b86e-78f8e40133b9 volumes: - node_outputs:/mnt/node_outputs - ${INPUT_train_dataset:?err}:/mnt/graph_inputs/train_dataset @@ -42,12 +42,12 @@ services: - -m - pirlib.backends.docker_batch - node - - gASVCAMAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMCGV2YWx1YXRllIwLZW50cnlwb2ludHOUfZSMBG1haW6UaACMCkVudHJ5cG9pbnSUk5QpgZR9lCiMB3ZlcnNpb26UjAJ2MZSMB2hhbmRsZXKUjBhleGFtcGxlLmV4YW1wbGU6ZXZhbHVhdGWUjAdydW50aW1llIwKcHl0aG9uOjMuOZSMB2NvZGV1cmyUTowFaW1hZ2WUjDFwaXJjbGktYnVpbGQ6MTJhYmVmZmYtODlkNy00ODNhLTliMjMtNzgxNTNkZDUxMGRjlHVic4wJZnJhbWV3b3JrlE6MBmNvbmZpZ5R9lIwGaW5wdXRzlF2UKGgAjAVJbnB1dJSTlCmBlH2UKGgFjBNrd2FyZ3MudGVzdF9kYXRhc2V0lIwGaW90eXBllIwJRElSRUNUT1JZlIwGc291cmNllGgAjApEYXRhU291cmNllJOUKYGUfZQojAdub2RlX2lklE6MC3N1YmdyYXBoX2lklE6MCW91dHB1dF9pZJROjA5ncmFwaF9pbnB1dF9pZJSMCXNlbnRlbmNlc5R1YowEbWV0YZRoAIwITWV0YWRhdGGUk5QpgZR9lCiMBG5hbWWUjBNrd2FyZ3MudGVzdF9kYXRhc2V0lIwLYW5ub3RhdGlvbnOUTnVidWJoHSmBlH2UKGgFjBJrd2FyZ3MucHJlZGljdGlvbnOUaCGMCURJUkVDVE9SWZRoI2glKYGUfZQoaCiMGGluZmVyX3BpcGVsaW5lLnNlbnRpbWVudJRoKU5oKowGcmV0dXJulGgrTnViaC1oLymBlH2UKGgyjBJrd2FyZ3MucHJlZGljdGlvbnOUaDROdWJ1YmWMB291dHB1dHOUXZRoAIwGT3V0cHV0lJOUKYGUfZQoaAWMBnJldHVybpRoIYwJREFUQUZSQU1FlGgtaC8pgZR9lChoMowGcmV0dXJulGg0TnVidWJhaC1oLymBlH2UKGgyjAhldmFsdWF0ZZRoNE51YnViLg== + - gASVCAMAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMCGV2YWx1YXRllIwLZW50cnlwb2ludHOUfZSMBG1haW6UaACMCkVudHJ5cG9pbnSUk5QpgZR9lCiMB3ZlcnNpb26UjAJ2MZSMB2hhbmRsZXKUjBhleGFtcGxlLmV4YW1wbGU6ZXZhbHVhdGWUjAdydW50aW1llIwKcHl0aG9uOjMuOZSMB2NvZGV1cmyUTowFaW1hZ2WUjDFwaXJjbGktYnVpbGQ6NjgyYzUyZjAtZDhhZS00M2FhLWI4NmUtNzhmOGU0MDEzM2I5lHVic4wJZnJhbWV3b3JrlE6MBmNvbmZpZ5R9lIwGaW5wdXRzlF2UKGgAjAVJbnB1dJSTlCmBlH2UKGgFjBNrd2FyZ3MudGVzdF9kYXRhc2V0lIwGaW90eXBllIwJRElSRUNUT1JZlIwGc291cmNllGgAjApEYXRhU291cmNllJOUKYGUfZQojAdub2RlX2lklE6MC3N1YmdyYXBoX2lklE6MCW91dHB1dF9pZJROjA5ncmFwaF9pbnB1dF9pZJSMCXNlbnRlbmNlc5R1YowEbWV0YZRoAIwITWV0YWRhdGGUk5QpgZR9lCiMBG5hbWWUjBNrd2FyZ3MudGVzdF9kYXRhc2V0lIwLYW5ub3RhdGlvbnOUTnVidWJoHSmBlH2UKGgFjBJrd2FyZ3MucHJlZGljdGlvbnOUaCGMCURJUkVDVE9SWZRoI2glKYGUfZQoaCiMGGluZmVyX3BpcGVsaW5lLnNlbnRpbWVudJRoKU5oKowGcmV0dXJulGgrTnViaC1oLymBlH2UKGgyjBJrd2FyZ3MucHJlZGljdGlvbnOUaDROdWJ1YmWMB291dHB1dHOUXZRoAIwGT3V0cHV0lJOUKYGUfZQoaAWMBnJldHVybpRoIYwJREFUQUZSQU1FlGgtaC8pgZR9lChoMowGcmV0dXJulGg0TnVidWJhaC1oLymBlH2UKGgyjAhldmFsdWF0ZZRoNE51YnViLg== - gASVJAEAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojAJpZJSMDXRyYWluX2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjARtZXRhlGgBjAhNZXRhZGF0YZSTlCmBlH2UKIwEbmFtZZSMDXRyYWluX2RhdGFzZXSUjAthbm5vdGF0aW9uc5ROdWJ1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZRoCmgMKYGUfZQoaA+MD3RyYW5zbGF0ZV9tb2RlbJRoEU51YnViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUaApoDCmBlH2UKGgPjAlzZW50ZW5jZXOUaBFOdWJ1YmUu depends_on: train_pipeline.infer_pipeline.sentiment: condition: service_completed_successfully - image: pircli-build:12abefff-89d7-483a-9b23-78153dd510dc + image: pircli-build:682c52f0-d8ae-43aa-b86e-78f8e40133b9 volumes: - node_outputs:/mnt/node_outputs - ${INPUT_sentences:?err}:/mnt/graph_inputs/sentences @@ -57,14 +57,14 @@ services: - -m - pirlib.backends.docker_batch - node - - gASV7QIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMGGluZmVyX3BpcGVsaW5lLnNlbnRpbWVudJSMC2VudHJ5cG9pbnRzlH2UjARtYWlulGgAjApFbnRyeXBvaW50lJOUKYGUfZQojAd2ZXJzaW9ulIwCdjGUjAdoYW5kbGVylIwZZXhhbXBsZS5leGFtcGxlOnNlbnRpbWVudJSMB3J1bnRpbWWUjApweXRob246My45lIwHY29kZXVybJROjAVpbWFnZZSMMXBpcmNsaS1idWlsZDoxMmFiZWZmZi04OWQ3LTQ4M2EtOWIyMy03ODE1M2RkNTEwZGOUdWJzjAlmcmFtZXdvcmuUTowGY29uZmlnlH2UjAZpbnB1dHOUXZQoaACMBUlucHV0lJOUKYGUfZQoaAWMBW1vZGVslIwGaW90eXBllIwERklMRZSMBnNvdXJjZZRoAIwKRGF0YVNvdXJjZZSTlCmBlH2UKIwHbm9kZV9pZJSMBXRyYWlulIwLc3ViZ3JhcGhfaWSUTowJb3V0cHV0X2lklIwGcmV0dXJulIwOZ3JhcGhfaW5wdXRfaWSUTnVijARtZXRhlGgAjAhNZXRhZGF0YZSTlCmBlH2UKIwEbmFtZZSMBW1vZGVslIwLYW5ub3RhdGlvbnOUTnVidWJoHSmBlH2UKGgFjAlzZW50ZW5jZXOUaCGMCURJUkVDVE9SWZRoI2glKYGUfZQoaCiMGmluZmVyX3BpcGVsaW5lLnRyYW5zbGF0ZV8xlGgqTmgrjAZyZXR1cm6UaC1OdWJoLmgwKYGUfZQoaDOMCXNlbnRlbmNlc5RoNU51YnViZYwHb3V0cHV0c5RdlGgAjAZPdXRwdXSUk5QpgZR9lChoBYwGcmV0dXJulGghjAlESVJFQ1RPUlmUaC5oMCmBlH2UKGgzjAZyZXR1cm6UaDVOdWJ1YmFoLmgwKYGUfZQoaDOMCXNlbnRpbWVudJRoNU51YnViLg== + - gASV7QIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMGGluZmVyX3BpcGVsaW5lLnNlbnRpbWVudJSMC2VudHJ5cG9pbnRzlH2UjARtYWlulGgAjApFbnRyeXBvaW50lJOUKYGUfZQojAd2ZXJzaW9ulIwCdjGUjAdoYW5kbGVylIwZZXhhbXBsZS5leGFtcGxlOnNlbnRpbWVudJSMB3J1bnRpbWWUjApweXRob246My45lIwHY29kZXVybJROjAVpbWFnZZSMMXBpcmNsaS1idWlsZDo2ODJjNTJmMC1kOGFlLTQzYWEtYjg2ZS03OGY4ZTQwMTMzYjmUdWJzjAlmcmFtZXdvcmuUTowGY29uZmlnlH2UjAZpbnB1dHOUXZQoaACMBUlucHV0lJOUKYGUfZQoaAWMBW1vZGVslIwGaW90eXBllIwERklMRZSMBnNvdXJjZZRoAIwKRGF0YVNvdXJjZZSTlCmBlH2UKIwHbm9kZV9pZJSMBXRyYWlulIwLc3ViZ3JhcGhfaWSUTowJb3V0cHV0X2lklIwGcmV0dXJulIwOZ3JhcGhfaW5wdXRfaWSUTnVijARtZXRhlGgAjAhNZXRhZGF0YZSTlCmBlH2UKIwEbmFtZZSMBW1vZGVslIwLYW5ub3RhdGlvbnOUTnVidWJoHSmBlH2UKGgFjAlzZW50ZW5jZXOUaCGMCURJUkVDVE9SWZRoI2glKYGUfZQoaCiMGmluZmVyX3BpcGVsaW5lLnRyYW5zbGF0ZV8xlGgqTmgrjAZyZXR1cm6UaC1OdWJoLmgwKYGUfZQoaDOMCXNlbnRlbmNlc5RoNU51YnViZYwHb3V0cHV0c5RdlGgAjAZPdXRwdXSUk5QpgZR9lChoBYwGcmV0dXJulGghjAlESVJFQ1RPUlmUaC5oMCmBlH2UKGgzjAZyZXR1cm6UaDVOdWJ1YmFoLmgwKYGUfZQoaDOMCXNlbnRpbWVudJRoNU51YnViLg== - gASVJAEAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojAJpZJSMDXRyYWluX2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjARtZXRhlGgBjAhNZXRhZGF0YZSTlCmBlH2UKIwEbmFtZZSMDXRyYWluX2RhdGFzZXSUjAthbm5vdGF0aW9uc5ROdWJ1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZRoCmgMKYGUfZQoaA+MD3RyYW5zbGF0ZV9tb2RlbJRoEU51YnViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUaApoDCmBlH2UKGgPjAlzZW50ZW5jZXOUaBFOdWJ1YmUu depends_on: train_pipeline.infer_pipeline.translate_1: condition: service_completed_successfully train_pipeline.train: condition: service_completed_successfully - image: pircli-build:12abefff-89d7-483a-9b23-78153dd510dc + image: pircli-build:682c52f0-d8ae-43aa-b86e-78f8e40133b9 volumes: - node_outputs:/mnt/node_outputs train_pipeline.infer_pipeline.translate_1: @@ -73,9 +73,9 @@ services: - -m - pirlib.backends.docker_batch - node - - gASV5QIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMGmluZmVyX3BpcGVsaW5lLnRyYW5zbGF0ZV8xlIwLZW50cnlwb2ludHOUfZSMBG1haW6UaACMCkVudHJ5cG9pbnSUk5QpgZR9lCiMB3ZlcnNpb26UjAJ2MZSMB2hhbmRsZXKUjBlleGFtcGxlLmV4YW1wbGU6dHJhbnNsYXRllIwHcnVudGltZZSMCnB5dGhvbjozLjmUjAdjb2RldXJslE6MBWltYWdllIwxcGlyY2xpLWJ1aWxkOjEyYWJlZmZmLTg5ZDctNDgzYS05YjIzLTc4MTUzZGQ1MTBkY5R1YnOMCWZyYW1ld29ya5ROjAZjb25maWeUfZSMA2tleZSMBXZhbHVllHOMBmlucHV0c5RdlChoAIwFSW5wdXSUk5QpgZR9lChoBYwGYXJncy4wlIwGaW90eXBllIwERklMRZSMBnNvdXJjZZRoAIwKRGF0YVNvdXJjZZSTlCmBlH2UKIwHbm9kZV9pZJROjAtzdWJncmFwaF9pZJROjAlvdXRwdXRfaWSUTowOZ3JhcGhfaW5wdXRfaWSUjA90cmFuc2xhdGVfbW9kZWyUdWKMBG1ldGGUaACMCE1ldGFkYXRhlJOUKYGUfZQojARuYW1llIwGYXJncy4wlIwLYW5ub3RhdGlvbnOUTnVidWJoHymBlH2UKGgFjAZhcmdzLjGUaCOMCURJUkVDVE9SWZRoJWgnKYGUfZQoaCpOaCtOaCxOaC2MCXNlbnRlbmNlc5R1YmgvaDEpgZR9lChoNIwGYXJncy4xlGg2TnVidWJljAdvdXRwdXRzlF2UaACMBk91dHB1dJSTlCmBlH2UKGgFjAZyZXR1cm6UaCOMCURJUkVDVE9SWZRoL2gxKYGUfZQoaDSMBnJldHVybpRoNk51YnViYWgvaDEpgZR9lChoNIwLdHJhbnNsYXRlXzGUaDZOdWJ1Yi4= + - gASV5QIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMGmluZmVyX3BpcGVsaW5lLnRyYW5zbGF0ZV8xlIwLZW50cnlwb2ludHOUfZSMBG1haW6UaACMCkVudHJ5cG9pbnSUk5QpgZR9lCiMB3ZlcnNpb26UjAJ2MZSMB2hhbmRsZXKUjBlleGFtcGxlLmV4YW1wbGU6dHJhbnNsYXRllIwHcnVudGltZZSMCnB5dGhvbjozLjmUjAdjb2RldXJslE6MBWltYWdllIwxcGlyY2xpLWJ1aWxkOjY4MmM1MmYwLWQ4YWUtNDNhYS1iODZlLTc4ZjhlNDAxMzNiOZR1YnOMCWZyYW1ld29ya5ROjAZjb25maWeUfZSMA2tleZSMBXZhbHVllHOMBmlucHV0c5RdlChoAIwFSW5wdXSUk5QpgZR9lChoBYwGYXJncy4wlIwGaW90eXBllIwERklMRZSMBnNvdXJjZZRoAIwKRGF0YVNvdXJjZZSTlCmBlH2UKIwHbm9kZV9pZJROjAtzdWJncmFwaF9pZJROjAlvdXRwdXRfaWSUTowOZ3JhcGhfaW5wdXRfaWSUjA90cmFuc2xhdGVfbW9kZWyUdWKMBG1ldGGUaACMCE1ldGFkYXRhlJOUKYGUfZQojARuYW1llIwGYXJncy4wlIwLYW5ub3RhdGlvbnOUTnVidWJoHymBlH2UKGgFjAZhcmdzLjGUaCOMCURJUkVDVE9SWZRoJWgnKYGUfZQoaCpOaCtOaCxOaC2MCXNlbnRlbmNlc5R1YmgvaDEpgZR9lChoNIwGYXJncy4xlGg2TnVidWJljAdvdXRwdXRzlF2UaACMBk91dHB1dJSTlCmBlH2UKGgFjAZyZXR1cm6UaCOMCURJUkVDVE9SWZRoL2gxKYGUfZQoaDSMBnJldHVybpRoNk51YnViYWgvaDEpgZR9lChoNIwLdHJhbnNsYXRlXzGUaDZOdWJ1Yi4= - gASVJAEAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojAJpZJSMDXRyYWluX2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjARtZXRhlGgBjAhNZXRhZGF0YZSTlCmBlH2UKIwEbmFtZZSMDXRyYWluX2RhdGFzZXSUjAthbm5vdGF0aW9uc5ROdWJ1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZRoCmgMKYGUfZQoaA+MD3RyYW5zbGF0ZV9tb2RlbJRoEU51YnViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUaApoDCmBlH2UKGgPjAlzZW50ZW5jZXOUaBFOdWJ1YmUu - image: pircli-build:12abefff-89d7-483a-9b23-78153dd510dc + image: pircli-build:682c52f0-d8ae-43aa-b86e-78f8e40133b9 volumes: - node_outputs:/mnt/node_outputs - ${INPUT_translate_model:?err}:/mnt/graph_inputs/translate_model @@ -86,12 +86,12 @@ services: - -m - pirlib.backends.docker_batch - node - - gASVswIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMBXRyYWlulIwLZW50cnlwb2ludHOUfZSMBG1haW6UaACMCkVudHJ5cG9pbnSUk5QpgZR9lCiMB3ZlcnNpb26UjAJ2MZSMB2hhbmRsZXKUjBVleGFtcGxlLmV4YW1wbGU6dHJhaW6UjAdydW50aW1llIwKcHl0aG9uOjMuOZSMB2NvZGV1cmyUTowFaW1hZ2WUjDFwaXJjbGktYnVpbGQ6MTJhYmVmZmYtODlkNy00ODNhLTliMjMtNzgxNTNkZDUxMGRjlHVic4wJZnJhbWV3b3JrlGgAjAlGcmFtZXdvcmuUk5QpgZR9lCiMBG5hbWWUjAdhZGFwdGRslGgOTowGY29uZmlnlE51YmgefZQojBRhZGFwdGRsL21pbl9yZXBsaWNhc5RLAYwUYWRhcHRkbC9tYXhfcmVwbGljYXOUSwR1jAZpbnB1dHOUXZRoAIwFSW5wdXSUk5QpgZR9lChoBYwHZGF0YXNldJSMBmlvdHlwZZSMCURJUkVDVE9SWZSMBnNvdXJjZZRoAIwKRGF0YVNvdXJjZZSTlCmBlH2UKIwHbm9kZV9pZJSMBWNsZWFulIwLc3ViZ3JhcGhfaWSUTowJb3V0cHV0X2lklIwGcmV0dXJulIwOZ3JhcGhfaW5wdXRfaWSUTnVijARtZXRhlGgAjAhNZXRhZGF0YZSTlCmBlH2UKGgcjAdkYXRhc2V0lIwLYW5ub3RhdGlvbnOUTnVidWJhjAdvdXRwdXRzlF2UaACMBk91dHB1dJSTlCmBlH2UKGgFjAZyZXR1cm6UaCmMBEZJTEWUaDZoOCmBlH2UKGgcjAZyZXR1cm6UaDxOdWJ1YmFoNmg4KYGUfZQoaByMBXRyYWlulGg8TnVidWIu + - gASVswIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMBXRyYWlulIwLZW50cnlwb2ludHOUfZSMBG1haW6UaACMCkVudHJ5cG9pbnSUk5QpgZR9lCiMB3ZlcnNpb26UjAJ2MZSMB2hhbmRsZXKUjBVleGFtcGxlLmV4YW1wbGU6dHJhaW6UjAdydW50aW1llIwKcHl0aG9uOjMuOZSMB2NvZGV1cmyUTowFaW1hZ2WUjDFwaXJjbGktYnVpbGQ6NjgyYzUyZjAtZDhhZS00M2FhLWI4NmUtNzhmOGU0MDEzM2I5lHVic4wJZnJhbWV3b3JrlGgAjAlGcmFtZXdvcmuUk5QpgZR9lCiMBG5hbWWUjAdhZGFwdGRslGgOTowGY29uZmlnlE51YmgefZQojBRhZGFwdGRsL21pbl9yZXBsaWNhc5RLAYwUYWRhcHRkbC9tYXhfcmVwbGljYXOUSwR1jAZpbnB1dHOUXZRoAIwFSW5wdXSUk5QpgZR9lChoBYwHZGF0YXNldJSMBmlvdHlwZZSMCURJUkVDVE9SWZSMBnNvdXJjZZRoAIwKRGF0YVNvdXJjZZSTlCmBlH2UKIwHbm9kZV9pZJSMBWNsZWFulIwLc3ViZ3JhcGhfaWSUTowJb3V0cHV0X2lklIwGcmV0dXJulIwOZ3JhcGhfaW5wdXRfaWSUTnVijARtZXRhlGgAjAhNZXRhZGF0YZSTlCmBlH2UKGgcjAdkYXRhc2V0lIwLYW5ub3RhdGlvbnOUTnVidWJhjAdvdXRwdXRzlF2UaACMBk91dHB1dJSTlCmBlH2UKGgFjAZyZXR1cm6UaCmMBEZJTEWUaDZoOCmBlH2UKGgcjAZyZXR1cm6UaDxOdWJ1YmFoNmg4KYGUfZQoaByMBXRyYWlulGg8TnVidWIu - gASVJAEAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojAJpZJSMDXRyYWluX2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjARtZXRhlGgBjAhNZXRhZGF0YZSTlCmBlH2UKIwEbmFtZZSMDXRyYWluX2RhdGFzZXSUjAthbm5vdGF0aW9uc5ROdWJ1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZRoCmgMKYGUfZQoaA+MD3RyYW5zbGF0ZV9tb2RlbJRoEU51YnViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUaApoDCmBlH2UKGgPjAlzZW50ZW5jZXOUaBFOdWJ1YmUu depends_on: train_pipeline.clean: condition: service_completed_successfully - image: pircli-build:12abefff-89d7-483a-9b23-78153dd510dc + image: pircli-build:682c52f0-d8ae-43aa-b86e-78f8e40133b9 volumes: - node_outputs:/mnt/node_outputs version: '3.9' diff --git a/example/example.py b/example/example.py index 3507677..0a6b774 100644 --- a/example/example.py +++ b/example/example.py @@ -14,7 +14,7 @@ def clean(dataset: DirectoryPath) -> DirectoryPath: with open(dataset / "file.txt") as f: print("clean({})".format(f.read().strip())) - outdir = task.context().output + outdir = clean.context.output with open(outdir / "file.txt", "w") as f: f.write("clean_result") return outdir @@ -22,12 +22,13 @@ def clean(dataset: DirectoryPath) -> DirectoryPath: @task(framework=AdaptDL(min_replicas=1, max_replicas=4)) def train(dataset: DirectoryPath) -> FilePath: - task_ctx = task.context() + task_ctx = train.context with open(dataset / "file.txt") as f: print("train({}, config={})".format(f.read().strip(), task_ctx.config)) - outfile = task.context().output + outfile = task_ctx.output with open(outfile, "w") as f: f.write("train_result") + task_ctx.logger.info(">>> Train accuracy: 0.83") return outfile @@ -46,28 +47,51 @@ def evaluate(kwargs: EvaluateInput) -> pandas.DataFrame: return df +class TranslateModel(object): + def translate(self, inp: str) -> str: + output = f"translation: {inp}" + return output + + @task def translate(args: Tuple[FilePath, DirectoryPath]) -> DirectoryPath: - model, sentences = args - task_ctx = task.context() - with open(model) as f, open(sentences / "file.txt") as g: + model_path, sentences = args + task_ctx = translate.context + model = task_ctx.translate_model + with open(model_path) as f, open(sentences / "file.txt") as g: + inp = g.read().strip() print( "translate({}, {}, config={})".format( f.read().strip(), - g.read().strip(), + inp, task_ctx.config) ) + translate_result = model.translate(inp) outdir = task_ctx.output with open(outdir / "file.txt", "w") as f: - f.write("translate_result") + f.write(translate_result) return outdir +@translate.setup +def translate_setup() -> None: + task_ctx = translate.context + task_ctx.translate_model = TranslateModel() + task_ctx.logger.info(">>> Initialized translation model.") + + +@translate.teardown +def translate_teardown() -> None: + task_ctx = translate.context + del task_ctx.translate_model + task_ctx.logger.info(">>> Cleaned up translation model.") + + @task def sentiment(model: FilePath, sentences: DirectoryPath) -> DirectoryPath: with open(model) as f, open(sentences / "file.txt") as g: print("sentiment({}, {})".format(f.read().strip(), g.read().strip())) - outdir = task.context().output + outdir = sentiment.context.output with open(outdir / "file.txt", "w") as f: f.write("sentiment_result") return outdir diff --git a/example/package_docker.yml b/example/package_docker.yml index e4501b1..d222453 100644 --- a/example/package_docker.yml +++ b/example/package_docker.yml @@ -8,7 +8,7 @@ graphs: handler: example.example:clean runtime: python:3.9 codeurl: null - image: pircli-build:12abefff-89d7-483a-9b23-78153dd510dc + image: pircli-build:682c52f0-d8ae-43aa-b86e-78f8e40133b9 framework: null config: {} inputs: @@ -38,7 +38,7 @@ graphs: handler: example.example:train runtime: python:3.9 codeurl: null - image: pircli-build:12abefff-89d7-483a-9b23-78153dd510dc + image: pircli-build:682c52f0-d8ae-43aa-b86e-78f8e40133b9 framework: name: adaptdl version: null @@ -72,7 +72,7 @@ graphs: handler: example.example:evaluate runtime: python:3.9 codeurl: null - image: pircli-build:12abefff-89d7-483a-9b23-78153dd510dc + image: pircli-build:682c52f0-d8ae-43aa-b86e-78f8e40133b9 framework: null config: {} inputs: @@ -112,7 +112,7 @@ graphs: handler: example.example:translate runtime: python:3.9 codeurl: null - image: pircli-build:12abefff-89d7-483a-9b23-78153dd510dc + image: pircli-build:682c52f0-d8ae-43aa-b86e-78f8e40133b9 framework: null config: key: value @@ -153,7 +153,7 @@ graphs: handler: example.example:sentiment runtime: python:3.9 codeurl: null - image: pircli-build:12abefff-89d7-483a-9b23-78153dd510dc + image: pircli-build:682c52f0-d8ae-43aa-b86e-78f8e40133b9 framework: null config: {} inputs: diff --git a/pirlib/backends/docker_batch.py b/pirlib/backends/docker_batch.py index c7fda49..26d4b4b 100644 --- a/pirlib/backends/docker_batch.py +++ b/pirlib/backends/docker_batch.py @@ -3,11 +3,13 @@ import pickle import sys import yaml +from dataclasses import asdict from typing import Optional import pirlib.pir from pirlib.backends import Backend from pirlib.handlers.v1 import HandlerV1Context, HandlerV1Event +from pirlib.utils import get_logger def encode(x): @@ -125,8 +127,13 @@ def run_node(node, graph_inputs): else: outputs[out.id] = None events = HandlerV1Event(inputs, outputs) - context = HandlerV1Context(node) + context = HandlerV1Context( + asdict(node), + get_logger(node.id), + ) + handler.setup_handler(context) handler.run_handler(events, context) + handler.teardown_handler(context) for out in node.outputs: path = f"/mnt/node_outputs/{node.id}/{out.id}" if out.iotype == "DATAFRAME": diff --git a/pirlib/backends/inproc.py b/pirlib/backends/inproc.py index 368347c..f766765 100644 --- a/pirlib/backends/inproc.py +++ b/pirlib/backends/inproc.py @@ -2,6 +2,7 @@ import importlib import pandas import tempfile +from dataclasses import asdict from typing import Any, Dict, Optional import pirlib.pir @@ -9,7 +10,7 @@ from pirlib.backends import Backend from pirlib.handlers.v1 import HandlerV1Context, HandlerV1Event from pirlib.iotypes import DirectoryPath, FilePath -from pirlib.utils import find_by_id +from pirlib.utils import find_by_id, get_logger class InprocBackend(Backend): @@ -91,6 +92,11 @@ def _execute_node(self, node: pirlib.pir.Node, inputs: Dict[str, Any]): else: outputs[out.id] = None event = HandlerV1Event(inputs, outputs) - context = HandlerV1Context(node) + context = HandlerV1Context( + asdict(node), + get_logger(node.id), + ) + handler.setup_handler(context) handler.run_handler(event, context) + handler.teardown_handler(context) return outputs diff --git a/pirlib/handlers/v1.py b/pirlib/handlers/v1.py index 997ea41..627e456 100644 --- a/pirlib/handlers/v1.py +++ b/pirlib/handlers/v1.py @@ -1,12 +1,13 @@ from abc import abstractmethod from dataclasses import dataclass -from typing import Any, Dict -from pirlib.pir import Node +from logging import Logger +from typing import Any, Dict, Optional @dataclass class HandlerV1Context(object): - node: Node + node: Dict[str, Any] + logger: Optional[Logger] = None @dataclass @@ -31,3 +32,15 @@ def run_handler( context: HandlerV1Context, ) -> None: raise NotImplementedError + + def setup_handler( + self, + context: HandlerV1Context, + ) -> None: + pass + + def teardown_handler( + self, + context: HandlerV1Context, + ) -> None: + pass diff --git a/pirlib/pipeline.py b/pirlib/pipeline.py index 7f439ca..1a57e64 100644 --- a/pirlib/pipeline.py +++ b/pirlib/pipeline.py @@ -75,14 +75,6 @@ def config(self): return self._config def __call__(self, *args, **kwargs): - if len(args) == 1 and callable(args[0]) and not kwargs: - wrapper = PipelineDefinition( - func=args[0], - name=self.name, - config=self.config, - ) - functools.update_wrapper(wrapper, args[0]) - return wrapper return self.instance(self.name)(*args, **kwargs) def instance(self, name: str) -> PipelineInstance: @@ -98,10 +90,16 @@ def pipeline( name: Optional[str] = None, config: Optional[dict] = None, ) -> PipelineDefinition: - wrapper = PipelineDefinition( - func=func, - name=name, - config=config, - ) - functools.update_wrapper(wrapper, func) - return wrapper + def wrapper(func) -> PipelineDefinition: + pipeline_dfn = PipelineDefinition( + func=func, + name=name, + config=config, + ) + functools.update_wrapper(pipeline_dfn, func) + return pipeline_dfn + + if func: + return wrapper(func) + else: + return wrapper diff --git a/pirlib/task.py b/pirlib/task.py index 744f0c8..34fc0d7 100644 --- a/pirlib/task.py +++ b/pirlib/task.py @@ -1,9 +1,9 @@ -import contextvars import copy import functools import inspect import typeguard from dataclasses import dataclass +from logging import Logger from typing import Any, Callable, Dict, Optional import pirlib.pir @@ -12,17 +12,23 @@ from pirlib.package import recurse_hint, task_call, package_task -_TASK_CONTEXT = contextvars.ContextVar("_TASK_CONTEXT") - - -@dataclass +@dataclass(init=False) class TaskContext: config: Dict[str, Any] output: Any + node_ctx: HandlerV1Context + logger: Optional[Logger] = None - -def task_context() -> TaskContext: - return _TASK_CONTEXT.get() + def __init__( + self, + config: Dict[str, Any], + output: Any, + node_ctx: HandlerV1Context, + ): + self.config = config + self.output = output + self.node_ctx = node_ctx + self.logger = node_ctx.logger class TaskInstance(object): @@ -78,10 +84,13 @@ def __init__( config: Optional[dict] = None, framework: Optional[pirlib.pir.Framework] = None, ): - self._func = func if func is None else typeguard.typechecked(func) + self._func = typeguard.typechecked(func) self._name = name if name else getattr(func, "__name__", None) self._config = copy.deepcopy(config) if config else None self._framework = framework + self._setup = None + self._teardown = None + self.context = None @property def func(self): @@ -100,15 +109,6 @@ def framework(self): return self._framework def __call__(self, *args, **kwargs): - if len(args) == 1 and callable(args[0]) and not kwargs: - wrapper = TaskDefinition( - func=args[0], - name=self.name, - config=self.config, - framework=self.framework, - ) - functools.update_wrapper(wrapper, args[0]) - return wrapper return self.instance(self.name)(*args, **kwargs) def instance(self, name: str) -> TaskInstance: @@ -130,10 +130,22 @@ def run_handler( ) -> None: inputs, outputs = event.inputs, event.outputs sig = inspect.signature(self.func) - task_context = TaskContext(context.node.config, None) - task_context.output = recurse_hint( - lambda name, hint: outputs[name], "return", sig.return_annotation - ) + if self.context is None: + # If setup_handler is not called, + # need to initialize task context here. + self.context = TaskContext( + context.node["config"], + recurse_hint( + lambda name, hint: outputs[name], + "return", + sig.return_annotation, + ), + context, + ) + else: + self.context.output = recurse_hint( + lambda name, hint: outputs[name], "return", sig.return_annotation + ) args, kwargs = [], {} for param in sig.parameters.values(): value = recurse_hint(lambda name, hint: inputs[name], param.name, param.annotation) @@ -141,11 +153,7 @@ def run_handler( kwargs[param.name] = value else: args.append(value) - token = _TASK_CONTEXT.set(task_context) - try: - return_value = self.func(*args, **kwargs) - finally: - _TASK_CONTEXT.reset(token) + return_value = self.func(*args, **kwargs) recurse_hint( lambda n, h, v: outputs.__setitem__(n, v), "return", @@ -153,6 +161,34 @@ def run_handler( return_value, ) + def setup_handler( + self, + context: HandlerV1Context, + ) -> None: + # Initialize task context + self.context = TaskContext( + context.node["config"], + None, + context, + ) + if self._setup: + self._setup() + + def teardown_handler( + self, + context: HandlerV1Context, + ) -> None: + if self._teardown: + self._teardown() + # Reset task context + self.context = None + + def setup(self, func) -> None: + setattr(self, "_setup", func) + + def teardown(self, func) -> None: + setattr(self, "_teardown", func) + def task( func: Optional[Callable] = None, @@ -167,14 +203,18 @@ def task( f_name = framework.name for k, v in framework.config.items(): config[f"{f_name}/{k}"] = v - wrapper = TaskDefinition( - func=func, - name=name, - config=config, - framework=framework, - ) - functools.update_wrapper(wrapper, func) - return wrapper + def wrapper(func) -> TaskDefinition: + task_dfn = TaskDefinition( + func=func, + name=name, + config=config, + framework=framework, + ) + functools.update_wrapper(task_dfn, func) + return task_dfn -task.context = task_context + if func: + return wrapper(func) + else: + return wrapper diff --git a/pirlib/utils.py b/pirlib/utils.py index aaa997c..d3aa0a7 100644 --- a/pirlib/utils.py +++ b/pirlib/utils.py @@ -1,4 +1,26 @@ +import logging +import sys + + def find_by_id(iterable, id): for item in iterable: if item.id == id: return item + + +FORMATTER = logging.Formatter("%(asctime)s — %(name)s — %(levelname)s — %(message)s") + + +def get_console_handler(): + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setFormatter(FORMATTER) + return console_handler + + +def get_logger(logger_name, logging_level=logging.INFO): + logger = logging.getLogger(logger_name) + logger.setLevel(logging_level) + if not logger.hasHandlers(): + logger.addHandler(get_console_handler()) + logger.propagate = False + return logger diff --git a/tests/test_example.py b/tests/test_example.py index e44a8eb..bcee3e2 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -12,7 +12,7 @@ def clean(dataset: DirectoryPath) -> DirectoryPath: with open(dataset / "file.txt") as f: print("clean({})".format(f.read().strip())) - outdir = task.context().output + outdir = clean.context.output with open(outdir / "file.txt", "w") as f: f.write("clean_result") return outdir @@ -20,11 +20,13 @@ def clean(dataset: DirectoryPath) -> DirectoryPath: @task(framework=AdaptDL(min_replicas=1, max_replicas=4)) def train(dataset: DirectoryPath) -> FilePath: + task_ctx = train.context with open(dataset / "file.txt") as f: - print("train({})".format(f.read().strip())) - outfile = task.context().output + print("train({}, config={})".format(f.read().strip(), task_ctx.config)) + outfile = task_ctx.output with open(outfile, "w") as f: f.write("train_result") + task_ctx.logger.info(">>> Train accuracy: 0.83") return outfile @@ -43,25 +45,46 @@ def evaluate(kwargs: EvaluateInput) -> pandas.DataFrame: return df +class TranslateModel(object): + def translate(self, inp: str) -> str: + output = f"translation: {inp}" + return output + + @task def translate(args: Tuple[FilePath, DirectoryPath]) -> DirectoryPath: - model, sentences = args - opctx = task.context() - with open(model) as f, open(sentences / "file.txt") as g: - print( - "translate({}, {}, config={})".format(f.read().strip(), g.read().strip(), opctx.config) - ) - outdir = opctx.output + model_path, sentences = args + task_ctx = translate.context + model = task_ctx.translate_model + with open(model_path) as f, open(sentences / "file.txt") as g: + inp = g.read().strip() + print("translate({}, {}, config={})".format(f.read().strip(), inp, task_ctx.config)) + translate_result = model.translate(inp) + outdir = task_ctx.output with open(outdir / "file.txt", "w") as f: - f.write("translate_result") + f.write(translate_result) return outdir +@translate.setup +def translate_setup() -> None: + task_ctx = translate.context + task_ctx.translate_model = TranslateModel() + task_ctx.logger.info(">>> Initialized translation model.") + + +@translate.teardown +def translate_teardown() -> None: + task_ctx = translate.context + del task_ctx.translate_model + task_ctx.logger.info(">>> Cleaned up translation model.") + + @task def sentiment(model: FilePath, sentences: DirectoryPath) -> DirectoryPath: with open(model) as f, open(sentences / "file.txt") as g: print("sentiment({}, {})".format(f.read().strip(), g.read().strip())) - outdir = task.context().output + outdir = sentiment.context.output with open(outdir / "file.txt", "w") as f: f.write("sentiment_result") return outdir @@ -69,9 +92,7 @@ def sentiment(model: FilePath, sentences: DirectoryPath) -> DirectoryPath: @pipeline def infer_pipeline( - translate_model: FilePath, - sentiment_model: FilePath, - sentences: DirectoryPath, + translate_model: FilePath, sentiment_model: FilePath, sentences: DirectoryPath ) -> DirectoryPath: translate_1 = translate.instance("translate_1") translate_1.config["key"] = "value" @@ -80,9 +101,7 @@ def infer_pipeline( @pipeline def train_pipeline( - train_dataset: DirectoryPath, - translate_model: FilePath, - sentences: DirectoryPath, + train_dataset: DirectoryPath, translate_model: FilePath, sentences: DirectoryPath ) -> Tuple[FilePath, pandas.DataFrame]: sentiment_model = train(clean(train_dataset)) sentiment = infer_pipeline(translate_model, sentiment_model, sentences)