diff --git a/cumulusci.yml b/cumulusci.yml index 64e8a897..23e0e8f5 100644 --- a/cumulusci.yml +++ b/cumulusci.yml @@ -65,29 +65,36 @@ tasks: num_records: 10 num_records_tablename: Account + generate_users_profiles_permission_sets: + class_path: cumulusci.tasks.bulkdata.generate_and_load_data_from_yaml.GenerateAndLoadDataFromYaml + options: + generator_yaml: examples/salesforce/ContentVersion.recipe.yml + flows: test_everything: steps: 1: task: generate_sf_accounts + 1.1: + task: generate_users_profiles_permission_sets 2: task: generate_sf_contacts 3: task: generate_sf_opportunities 4: - flow: npsp:install_prod + task: generate_content_documents 5: - task: generate_npsp_accounts + flow: npsp:install_prod 6: - task: generate_npsp_contacts + task: generate_npsp_accounts 7: - task: generate_npsp_opportunities + task: generate_npsp_contacts 8: - task: generate_opportunity_contact_roles + task: generate_npsp_opportunities 9: - task: generate_opportunities_and_contacts + task: generate_opportunity_contact_roles 10: - task: generate_content_documents + task: generate_opportunities_and_contacts contacts_for_accounts: steps: diff --git a/docs/extending.md b/docs/extending.md index bf1cce54..59f16fc4 100644 --- a/docs/extending.md +++ b/docs/extending.md @@ -114,7 +114,9 @@ class PluginThatCounts(SnowfakeryPlugin): ``` Plugins also have access to a dictionary called `self.context.field_vars()` which -represents the values that would be available to a formula running in the same context. +represents the values that would be available to a formula running in the same context +and `self.context.current_filename` which is the filename of the YAML file being +processed. Plugins can return normal Python primitive types, `datetime.date`, `ObjectRow` or `PluginResult` objects. `ObjectRow` objects represent new output records/objects. `PluginResult` objects expose a namespace that other code can access through dot-notation. `PluginResult` instances can be diff --git a/docs/salesforce.md b/docs/salesforce.md index a25ee575..5e2b6960 100644 --- a/docs/salesforce.md +++ b/docs/salesforce.md @@ -54,10 +54,14 @@ $ cci flow run test_everything ## Incorporating Information from Salesforce There are various cases where it might be helpful to relate newly created synthetic -data to existing data in a Salesforce org. For example, that data might have -been added in a previous CumulusCI task or some other process. +data to existing data in a Salesforce org. Perhaps that data was added +in a previous CumulusCI task or some other process. If your use cases are +intensive, please remember to read the section +[A Note On Performance](#a-note-on-performance) -For example, if you have a Campaign object and would like to associate +### SalesforceQuery.find_record and SalesforceQuery.random_record + +Let's use an example where you have a Campaign object and would like to associate Contacts to it through CampaignMembers. Here is an example where we query a particular Campaign object: @@ -113,6 +117,11 @@ NOTE: The features we are discussing in this section are for linking to records that are in the Salesforce org _before_ the recipe iteration started. These features are not for linking to records created by the recipe itself. +In extremely large loads, CumulusCI can be configured to upload "portions" or "batches" of +records from the recipe. Previous portions _are_ in the org and therefore can be queried. + +### Querying Field Data + Sometimes we want to do more than just link to the other record. For example, perhaps we want to create Users for Contacts and have the Users have the same name as the Contacts. @@ -152,6 +161,8 @@ have Contacts, but you can see how you would connect a synthetic object to a pre-existing object, while also getting access to other fields. +### SOQL Datasets + If you would like to use Salesforce query as a Dataset, that's another way that you can ensure that every synthetic record you create is associated with a distinct record from Salesforce. @@ -319,24 +330,30 @@ There is also an alternate syntax which allows nicknaming: Files can be used as Salesforce ContentVersions like this: ```yaml -- plugin: snowfakery.standard_plugins.base64.Base64 -- plugin: snowfakery.standard_plugins.file.File +# examples/salesforce/ContentVersion.recipe.yml +- plugin: snowfakery.standard_plugins.Salesforce - object: Account nickname: FileOwner fields: Name: fake: company - object: ContentVersion - nickname: FileAttachment fields: Title: Attachment for ${{Account.Name}} PathOnClient: example.pdf - Description: example.pdf + Description: The example.pdf file VersionData: - Base64.encode: - - File.file_data: - encoding: binary - file: ${{PathOnClient}} + Salesforce.ContentFile: + file: example.pdf FirstPublishLocationId: reference: Account ``` + +### A Note On Performance + +Calling into Salesforce is much slower than most things you do in +Snowfakery. The network traffic alone is comparatively slow. + +Consider using [variables](index.md#defining-variables) to "remember" +data you've queried, and if you need to pull down a lot of data, use +a SOQL Dataset which will intrinsically cache the data for you. diff --git a/examples/base64_file.recipe.yml b/examples/base64_file.recipe.yml new file mode 100644 index 00000000..8319a0e4 --- /dev/null +++ b/examples/base64_file.recipe.yml @@ -0,0 +1,32 @@ +- plugin: snowfakery.standard_plugins.base64.Base64 +- plugin: snowfakery.standard_plugins.file.File +- plugin: snowfakery.standard_plugins.Salesforce +- object: Account + nickname: FileOwner + fields: + Name: + fake: company +- object: ContentVersion + nickname: FileAttachment + fields: + Title: Attachment for ${{Account.Name}} + PathOnClient: salesforce/example.pdf + Description: example.pdf + VersionData: + Base64.encode: + - File.file_data: + encoding: binary + file: ${{PathOnClient}} + FirstPublishLocationId: + reference: Account +- object: ContentVersion + nickname: FileAttachment2 + fields: + Title: Attachment for ${{Account.Name}} + PathOnClient: salesforce/example.pdf + Description: example.pdf + VersionData: + Salesforce.ContentFile: + file: ${{PathOnClient}} + FirstPublishLocationId: + reference: Account diff --git a/examples/salesforce/CommunityUsers.yml b/examples/salesforce/CommunityUsers.yml deleted file mode 100644 index c181aaf5..00000000 --- a/examples/salesforce/CommunityUsers.yml +++ /dev/null @@ -1,88 +0,0 @@ -- plugin: snowfakery.standard_plugins.Salesforce -- plugin: snowfakery.standard_plugins.Salesforce.SalesforceQuery - -- object: User - fields: - Alias: Grace - Username: - fake: Username - LastName: Wong - Email: ${{Username}} - TimeZoneSidKey: America/Bogota - LocaleSidKey: en_US - EmailEncodingKey: UTF-8 - LanguageLocaleKey: en_US - ProfileId: - Salesforce.ProfileId: Identity User - joins_from: - - object: PermissionSetAssignment - join_field: AssigneeId - to: - - PermissionSetId: - Salesforce.PermissionSet: ActionPlans - - PermissionSetId: - Salesforce.PermissionSet: VoiceInbound - - PermissionSetId: - Salesforce.PermissionSet: DocumentChecklist - - friends: - - object: PermissionSetAssignment - fields: - AssigneeId: - reference: User - PermissionSetId: - SalesforceQuery.find_record: - from: PermissionSet - where: Name='ActionPlans' - - object: PermissionSetAssignment - fields: - AssigneeId: - reference: User - PermissionSetId: - SalesforceQuery.find_record: - from: PermissionSet - where: Name='ActionPlans' - - # __permissionSets: - # Salesforce.PermissionSetAssignments: - # names: ActionPlans,CallCoachingUser - # - object: __junk_wrapper - # friends: - # - object: PermissionSetAssignment - # fields: - # AssigneeId: - # reference: User - # PermissionSetId: - # SalesforceQuery.find_record: - # query_from: PermissionSet where Name='ActionPlans' - # - object: PermissionSetAssignment - # fields: - # AssigneeId: - # reference: User - # PermissionSetId: - # SalesforceQuery.find_record: - # query_from: PermissionSet where Name='ActionPlans' - # - object: PermissionSetAssignment - - # fields: - # AssigneeId: - # reference: User - # PermissionSetId: - # SalesforceQuery.find_record: - # query_from: PermissionSet where Name='CallCoachingUser' -- object: User - nickname: RandomizedUser - fields: - Username: - fake: Username - LastName: - fake: last_name - Email: - fake: email - Alias: Grace - TimeZoneSidKey: America/Bogota - LocaleSidKey: en_US - EmailEncodingKey: UTF-8 - LanguageLocaleKey: en_US - ProfileId: - Salesforce.ProfileId: Identity User diff --git a/examples/salesforce/ContentVersion.recipe.yml b/examples/salesforce/ContentVersion.recipe.yml index 01abf314..0994eb19 100644 --- a/examples/salesforce/ContentVersion.recipe.yml +++ b/examples/salesforce/ContentVersion.recipe.yml @@ -1,20 +1,16 @@ -- plugin: snowfakery.standard_plugins.base64.Base64 -- plugin: snowfakery.standard_plugins.file.File +- plugin: snowfakery.standard_plugins.Salesforce - object: Account nickname: FileOwner fields: Name: fake: company - object: ContentVersion - nickname: FileAttachment fields: Title: Attachment for ${{Account.Name}} PathOnClient: example.pdf - Description: example.pdf + Description: The example.pdf file VersionData: - Base64.encode: - - File.file_data: - encoding: binary - file: ${{PathOnClient}} + Salesforce.ContentFile: + file: example.pdf FirstPublishLocationId: reference: Account diff --git a/examples/salesforce/UsersProfilesPermissions.yml b/examples/salesforce/UsersProfilesPermissions.yml new file mode 100644 index 00000000..1e1d7260 --- /dev/null +++ b/examples/salesforce/UsersProfilesPermissions.yml @@ -0,0 +1,50 @@ +- plugin: snowfakery.standard_plugins.Salesforce +- plugin: snowfakery.standard_plugins.Salesforce.SalesforceQuery + +- object: User + fields: + Alias: Grace + Username: + fake: Username + LastName: Wong + Email: ${{Username}} + TimeZoneSidKey: America/Bogota + LocaleSidKey: en_US + EmailEncodingKey: UTF-8 + LanguageLocaleKey: en_US + ProfileId: + Salesforce.ProfileId: Identity User + + friends: + - object: PermissionSetAssignment + fields: + AssigneeId: + reference: User + PermissionSetId: + SalesforceQuery.find_record: + from: PermissionSet + where: Name='CommerceUser' + - object: PermissionSetAssignment + fields: + AssigneeId: + reference: User + PermissionSetId: + SalesforceQuery.find_record: + from: PermissionSet + where: Name='SalesConsoleUser' +- object: User + nickname: RandomizedUser + fields: + Username: + fake: Username + LastName: + fake: last_name + Email: + fake: email + Alias: Grace + TimeZoneSidKey: America/Bogota + LocaleSidKey: en_US + EmailEncodingKey: UTF-8 + LanguageLocaleKey: en_US + ProfileId: + Salesforce.ProfileId: Identity User diff --git a/snowfakery/plugins.py b/snowfakery/plugins.py index 03256cc1..ae4fd32a 100644 --- a/snowfakery/plugins.py +++ b/snowfakery/plugins.py @@ -119,6 +119,10 @@ def evaluate(self, field_definition): else: raise f"Cannot simplify {field_definition}. Perhaps should have used evaluate_raw?" + @property + def current_filename(self): + return self.interpreter.current_context.current_template.filename + def lazy(func: Any) -> Callable: """A lazy function is one that expects its arguments to be unparsed""" diff --git a/snowfakery/standard_plugins/Salesforce.py b/snowfakery/standard_plugins/Salesforce.py index 9eb0a5a9..2f833256 100644 --- a/snowfakery/standard_plugins/Salesforce.py +++ b/snowfakery/standard_plugins/Salesforce.py @@ -2,6 +2,7 @@ from logging import getLogger from tempfile import TemporaryDirectory from pathlib import Path +from base64 import b64encode from snowfakery.plugins import ParserMacroPlugin from snowfakery.data_generator_runtime_object_model import ( @@ -266,60 +267,6 @@ def _parse_special_args(self, args): return sobj, nickname - # FIXME: This code is not documented or tested - def ContentFile(self, context, args) -> ObjectTemplate: - return { - "Base64.encode": [ - {"File.file_data": {"encoding": "binary", "file": args.get("path")}} - ] - } - - def PermissionSetAssignments(self, context, args) -> ObjectTemplate: - names = args.get("names") - if not isinstance(names, str): - raise DataGenValueError( - f"string `names` not specified for PermissionSetAssignments: {names}" - ) - names = names.split(",") - line_info = context.line_num() - decls = [self._generate_psa(context, line_info, name) for name in names] - - return ObjectTemplate( - "__wrapper_for_permission_sets", - filename=line_info["filename"], - line_num=line_info["line_num"], - friends=decls, - ) - - def _generate_psa(self, context, line_info, name): - fields = {"AssigneeId": ("Use")} - - query = f"PermissionSet where Name = '{name}'" - - fields = [ - FieldFactory( - "PermissionSetId", - StructuredValue( - "SalesforceQuery.find_record", {"from": query}, **line_info - ), - **line_info, - ), - FieldFactory( - "AssigneeId", - StructuredValue("reference", ["User"], **line_info), - **line_info, - ), - ] - - new_template = ObjectTemplate( - "PermissionSetAssignment", - filename=line_info["filename"], - line_num=line_info["line_num"], - fields=fields, - ) - context.register_template(new_template) - return new_template - class Functions: def ProfileId(self, name): query = f"select Id from Profile where Name='{name}'" @@ -327,6 +274,12 @@ def ProfileId(self, name): Profile = ProfileId + def ContentFile(self, file: str): + template_path = Path(self.context.current_filename).parent + + with open(template_path / file, "rb") as data: + return b64encode(data.read()).decode("ascii") + # TODO: Tests for this class class SOQLDatasetImpl(DatasetBase): diff --git a/snowfakery/standard_plugins/file.py b/snowfakery/standard_plugins/file.py index cfdf4122..c5422c24 100644 --- a/snowfakery/standard_plugins/file.py +++ b/snowfakery/standard_plugins/file.py @@ -8,9 +8,7 @@ def file_data(self, file, encoding="utf-8"): if encoding == "binary": encoding = "latin-1" - context_filename = Path( - self.context.interpreter.current_context.current_template.filename - ).parent + template_path = Path(self.context.current_filename).parent - with open(context_filename / file, "rb") as data: + with open(template_path / file, "rb") as data: return data.read().decode(encoding) diff --git a/tests/cassettes/TestSOQLDatasets.test_dataset_bad_query_bulk.yaml b/tests/cassettes/TestSOQLDatasets.test_dataset_bad_query_bulk.yaml new file mode 100644 index 00000000..9d6bd867 --- /dev/null +++ b/tests/cassettes/TestSOQLDatasets.test_dataset_bad_query_bulk.yaml @@ -0,0 +1,132 @@ +interactions: +- request: + body: queryAccountCSV + headers: + Request-Headers: + - Elided + method: POST + uri: https://orgname.my.salesforce.com/services/async/50.0/job + response: + body: + string: "\n + 7500R000004aJJGQA2\n query\n Account\n + 0050R000008NsbLQAS\n 2021-06-21T05:38:02.000Z\n + 2021-06-21T05:38:02.000Z\n Open\n + Parallel\n CSV\n + 0\n 0\n + 0\n 0\n + 0\n 0\n + 0\n 50.0\n 0\n + 0\n 0\n + 0\n" + headers: + Content-Type: + - application/xml + Response-Headers: SF-Elided + status: + code: 201 + message: Created +- request: + body: 'SELECT Xyzzy FROM Account ' + headers: + Request-Headers: + - Elided + method: POST + uri: https://orgname.my.salesforce.com/services/async/50.0/job/7500R000004aJJGQA2/batch + response: + body: + string: "\n + 7510R000004aCMsQAM\n 7500R000004aJJGQA2\n Queued\n + 2021-06-21T05:38:02.000Z\n 2021-06-21T05:38:02.000Z\n + 0\n 0\n + 0\n 0\n + 0\n" + headers: + Content-Type: + - application/xml + Response-Headers: SF-Elided + status: + code: 201 + message: Created +- request: + body: null + headers: + Request-Headers: + - Elided + method: GET + uri: https://orgname.my.salesforce.com/services/async/50.0/job/7500R000004aJJGQA2 + response: + body: + string: "\n + 7500R000004aJJGQA2\n query\n Account\n + 0050R000008NsbLQAS\n 2021-06-21T05:38:02.000Z\n + 2021-06-21T05:38:02.000Z\n Open\n + Parallel\n CSV\n + 0\n 0\n + 0\n 1\n + 1\n 0\n + 0\n 50.0\n 0\n + 0\n 0\n + 0\n" + headers: + Content-Type: + - application/xml + Response-Headers: SF-Elided + status: + code: 200 + message: OK +- request: + body: null + headers: + Request-Headers: + - Elided + method: GET + uri: https://orgname.my.salesforce.com/services/async/50.0/job/7500R000004aJJGQA2/batch + response: + body: + string: "\n + \n 7510R000004aCMsQAM\n 7500R000004aJJGQA2\n + \ Failed\n InvalidBatch : Failed to process + query: INVALID_FIELD: SELECT Xyzzy FROM Account ^ ERROR at Row:1:Column:8 + No such column 'Xyzzy' on entity 'Account'. If you are attempting to use a + custom field, be sure to append the '__c' after the custom field name. Please + reference your WSDL or the describe call for the appropriate names.\n + \ 2021-06-21T05:38:02.000Z\n 2021-06-21T05:38:03.000Z\n + \ 0\n 0\n + \ 0\n 0\n + \ 0\n \n" + headers: + Content-Type: + - application/xml + Response-Headers: SF-Elided + status: + code: 200 + message: OK +- request: + body: Closed + headers: + Request-Headers: + - Elided + method: POST + uri: https://orgname.my.salesforce.com/services/async/50.0/job/7500R000004aJJGQA2 + response: + body: + string: "\n + 7500R000004aJJGQA2\n query\n Account\n + 0050R000008NsbLQAS\n 2021-06-21T05:38:02.000Z\n + 2021-06-21T05:38:02.000Z\n Closed\n + Parallel\n CSV\n + 0\n 0\n + 0\n 1\n + 1\n 0\n + 0\n 50.0\n 0\n + 0\n 0\n + 0\n" + headers: + Content-Type: + - application/xml + Response-Headers: SF-Elided + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestSalesforcePlugin.test_profile_id.yaml b/tests/cassettes/TestSalesforcePlugin.test_profile_id.yaml new file mode 100644 index 00000000..89f4d353 --- /dev/null +++ b/tests/cassettes/TestSalesforcePlugin.test_profile_id.yaml @@ -0,0 +1,21 @@ +interactions: +- request: + body: null + headers: + Request-Headers: + - Elided + method: GET + uri: https://orgname.my.salesforce.com/services/data/v50.0/query/?q=select+Id+from+Profile+where+Name%3D%27Identity+User%27 + response: + body: + string: "{\n \"totalSize\" : 1,\n \"done\" : true,\n \"records\" : [ {\n + \ \"attributes\" : {\n \"type\" : \"Profile\",\n \"url\" : \"/services/data/v50.0/sobjects/Profile/00e0R000000yGOcQAM\"\n + \ },\n \"Id\" : \"00e0R000000yGOcQAM\"\n } ]\n}" + headers: + Content-Type: + - application/json;charset=UTF-8 + Response-Headers: SF-Elided + status: + code: 200 + message: OK +version: 1 diff --git a/tests/conftest_extras_w_cci.py b/tests/conftest_extras_w_cci.py index 72d051f0..637b5144 100644 --- a/tests/conftest_extras_w_cci.py +++ b/tests/conftest_extras_w_cci.py @@ -66,7 +66,6 @@ def no_services(*args, **kwargs): # TODO: Port this back to CCI def sf_before_record_response(response): # salesforce_bulk needs the Content-Type header. - print(response["headers"]) content_type = response["headers"].get("Content-Type") response["headers"] = { "Response-Headers": "SF-Elided", diff --git a/tests/test_salesforce_gen.py b/tests/test_salesforce_gen.py index 0aed5d98..d5fabaaa 100644 --- a/tests/test_salesforce_gen.py +++ b/tests/test_salesforce_gen.py @@ -1,13 +1,43 @@ from base64 import b64decode +from io import StringIO +import pytest from snowfakery import generate_data +from snowfakery.standard_plugins.Salesforce import SalesforceConnection +from snowfakery import data_gen_exceptions as exc +from tests.test_with_cci import skip_if_cumulusci_missing class TestSalesforceGen: def test_content_version(self, generated_rows): - content_version = "examples/salesforce/ContentVersion.recipe.yml" + content_version = "examples/base64_file.recipe.yml" generate_data(content_version) - b64data = generated_rows.table_values("ContentVersion", 0)["VersionData"] - rawdata = b64decode(b64data) - assert rawdata.startswith(b"%PDF-1.3") - assert b"Helvetica" in rawdata + for i in range(0, 2): + b64data = generated_rows.table_values("ContentVersion", i)["VersionData"] + rawdata = b64decode(b64data) + assert rawdata.startswith(b"%PDF-1.3") + assert b"Helvetica" in rawdata + + +class TestSalesforceConnection: + def test_bad_kwargs(self): + sfc = SalesforceConnection(None) + with pytest.raises(exc.DataGenError, match="Unknown argument"): + sfc.compose_query( + "context_name", fields=["blah"], xyzzy="foo", **{"from": "blah"} + ) + + +class TestSalesforcePlugin: + @skip_if_cumulusci_missing + @pytest.mark.vcr() + def test_profile_id(self, generated_rows, org_config): + yaml = """ + - plugin: snowfakery.standard_plugins.Salesforce + - object: foo + fields: + ProfileId: + Salesforce.ProfileId: Identity User + """ + generate_data(StringIO(yaml), plugin_options={"org_name": org_config.name}) + assert generated_rows.table_values("foo", 0, "ProfileId").startswith("00e") diff --git a/tests/test_with_cci.py b/tests/test_with_cci.py index 49b549d5..a31893d2 100644 --- a/tests/test_with_cci.py +++ b/tests/test_with_cci.py @@ -269,8 +269,6 @@ def test_find_records_returns_multiple(self, org_config, sf, generated_rows): assert generated_rows.mock_calls[0][1][1]["AccountId"] == first_user_id -# TODO: add tests for SOQLDatasets -# ensure that all documented params/methods are covered. @skip_if_cumulusci_missing class TestSOQLDatasets: @pytest.mark.vcr() @@ -401,6 +399,27 @@ def test_dataset_bad_query(self, org_config, sf, generated_rows): with pytest.raises(DataGenError, match="Xyzzy"): generate_data(StringIO(yaml), plugin_options={"org_name": org_config.name}) + @pytest.mark.vcr() + @patch( + "simple_salesforce.Salesforce.restful", + return_value={ + "sObjects": [{"name": "Account", "count": 3000}] + }, # forces bulk mode + ) + def test_dataset_bad_query_bulk(self, restful, org_config): + yaml = """ +- plugin: snowfakery.standard_plugins.Salesforce.SOQLDataset +- object: Contact + count: 10 + fields: + __users_from_salesforce: + SOQLDataset.shuffle: + fields: Xyzzy + from: Account + """ + with pytest.raises(DataGenError, match="No such column 'Xyzzy' on entity"): + generate_data(StringIO(yaml), plugin_options={"org_name": org_config.name}) + def test_dataset_no_fields(self, org_config, sf, generated_rows): yaml = """ - plugin: snowfakery.standard_plugins.Salesforce.SOQLDataset