CWR V2.2 support: parser, encoder, and config updates

README.rst

@@ -1,7 +1,7 @@
 CWR Data Model API
 ==================
 
-This projects offers a domain model for the CISAC CWR standard v2.1 to be
+This projects offers a domain model for the CISAC CWR standard v2.2 to be
 used on Python applications, along a series of parsing which allow
 transformations between the model and various data structures.
 
@@ -13,6 +13,9 @@ works data.
 While the CWR standard has been created by `CISAC`_ this library has been
 developed by `WESO`_ independently, with help from `BMAT`_.
 
+This is a fork of the `official CWR-DataApi repository`_ by WESO, updated to
+support CWR standard v2.2 by `max1millions`_.
+
 .. image:: https://badge.fury.io/py/cwr-api.svg
     :target: https://pypi.python.org/pypi/cwr-api
     :alt: CWR-API Pypi package page
@@ -51,9 +54,10 @@ Prerequisites
 
 The project has been tested in the following versions of the interpreter:
 
-- Python 3.4
-- Python 3.5
-- Python 3.6
+- Python 3.9
+- Python 3.10
+- Python 3.11
+- Python 3.12
 
 All other dependencies are indicated on the requirements.txt file.
 The included makefile can install them with the command:
@@ -146,9 +150,11 @@ The project has been released under the `MIT License`_.
 .. _BMAT: http://www.bmat.com/
 .. _WESO: http://www.weso.es/
 .. _project issues page: https://github.com/weso/CWR-DataApi/issues
-.. _Pyparsing: https://pyparsing.wikispaces.com/
+.. _Pyparsing: https://pyparsing-docs.readthedocs.io/
 .. _Pypi package: https://pypi.python.org/pypi/CWR-API
 .. _Sphinx: http://sphinx-doc.org/
 .. _latest docs: http://cwr-dataapi.readthedocs.org
 .. _GitHub project page: https://github.com/weso/CWR-DataApi
+.. _official CWR-DataApi repository: https://github.com/weso/CWR-DataApi
+.. _max1millions: https://github.com/max1millions
 .. _MIT License: http://www.opensource.org/licenses/mit-license.php

config_cwr/accessor.py

@@ -43,7 +43,7 @@ def read_yaml_file(self, file_name):
         """
         with open(os.path.join(self.__path(), os.path.basename(file_name)),
                   'rt') as yamlfile:
-            return yaml.load(yamlfile)
+            return yaml.load(yamlfile, Loader=yaml.FullLoader)
 
     def read_config_file(self, file_name):
         """

config_cwr/default_values.yml

@@ -1 +1 @@
-default_version: 2.1
+default_version: 2.2

config_cwr/field_config_common.yml

@@ -137,7 +137,7 @@ date_publication_printed_edition:
 
 duration:
   type: time
-  size: 8
+  size: 6
   name: Duration
 
 ean13:
@@ -153,6 +153,12 @@ edi_standard:
   values:
     - '01.10'
 
+hdr_version_type:
+  type: alphanum
+  size: 3
+  name: Header Version Type
+  results_name: version_type
+
 entire_work_title:
   type: alphanum
   size: 60
@@ -599,7 +605,7 @@ submitter_work_n:
   name: Submitter Work Number
 
 tax_id:
-  type: numeric
+  type: alphanum
   size: 9
   name: Tax ID Number
 
@@ -765,4 +771,34 @@ writer_unknown:
 year_production:
   type: numeric
   size: 4
-  name: Year of Production
+  name: Year of Production
+
+recording_title:
+  type: alphanum_ext
+  size: 60
+  name: Recording Title
+
+version_title:
+  type: alphanum_ext
+  size: 60
+  name: Version Title
+
+display_artist:
+  type: alphanum_ext
+  size: 60
+  name: Display Artist
+
+record_label:
+  type: alphanum
+  size: 60
+  name: Record Label
+
+isrc_validity:
+  type: alphanum
+  size: 20
+  name: ISRC Validity
+
+submitter_recording_identifier:
+  type: alphanum_end
+  size: 14
+  name: Submitter Recording Identifier

config_cwr/record_config_common.cml

@@ -340,7 +340,7 @@ transaction_record:
 
 transaction_record:
     id: publisher_territory
-    head: SPT
+    head: SPT,OPT
     rules:
       [
         field: ip_n
@@ -396,6 +396,14 @@ transaction_record:
         optional [
             field: media_type
         ]
+        optional [
+            field: recording_title
+            field: version_title
+            field: display_artist
+            field: record_label
+            field: isrc_validity
+            field: submitter_recording_identifier
+        ]
       ]
 
 transaction_record:
@@ -422,6 +430,7 @@ record:
         optional
         [
           field: character_set
+          field: hdr_version_type
         ]
       ]
 
@@ -573,6 +582,7 @@ transaction_record:
         field: submitter_agreement_n
         field: society_assigned_agreement_n
         field: writer_ip_n
+        field: publisher_sequence_n
       ]
 
 transaction_record:

cwr/grammar/factory/config.py

@@ -15,7 +15,7 @@
     pp.Word(pp.nums).setResultsName('count')
 rule_at_least.setParseAction(lambda v: int(v[0]))
 
-_rule_config_string = pp.Regex('[^()\[\]\\n,:]*')
+_rule_config_string = pp.Regex(r'[^()\[\]\n,:]*')
 _rule_config_string.setParseAction(lambda v: v[0].strip())
 
 _rule_identifier = _rule_config_string + pp.Literal(':').suppress()

cwr/grammar/field/basic.py

@@ -3,7 +3,6 @@
 import datetime
 
 import pyparsing as pp
-from pyparsing import ParseResults
 
 """
 CWR fields grammar.

cwr/group.py

@@ -44,7 +44,7 @@ def __init__(self,
                  record_type='',
                  group_id=0,
                  transaction_type='',
-                 version_number='02.10',
+                 version_number='02.20',
                  batch_request_id=0
                  ):
         super(GroupHeader, self).__init__(

cwr/interested_party.py

@@ -115,7 +115,7 @@ def __init__(self,
                  record_type='',
                  transaction_sequence_n=0,
                  record_sequence_n=0,
-                 first_recording_refusal='U',
+                 first_recording_refusal='N',
                  usa_license='',
                  pr_society=None,
                  pr_ownership_share=0,
@@ -292,7 +292,7 @@ def usa_license(self, value):
 
 class IPTerritoryOfControlRecord(TransactionRecord):
     """
-    Represents a CWR Publisher or Writer Territory of Control (SPT/SWT).
+    Represents a CWR Publisher or Writer Territory of Control (SPT/OPT/SWT).
 
     This indicates if a Publisher or Writer has control or not over a
     Territory, and the shares it has on it.
@@ -551,7 +551,7 @@ def __init__(self,
                  sr_society=None,
                  sr_ownership_share=0,
                  special_agreements=None,
-                 first_recording_refusal='U',
+                 first_recording_refusal='N',
                  usa_license=''
                  ):
         """
@@ -995,7 +995,8 @@ def __init__(self,
                  publisher_name='',
                  writer_ip_n='',
                  submitter_agreement_n=None,
-                 society_assigned_agreement_n=None
+                 society_assigned_agreement_n=None,
+                 publisher_sequence_n=0
                  ):
         super(PublisherForWriterRecord, self).__init__(
             record_type,
@@ -1011,6 +1012,9 @@ def __init__(self,
         self._submitter_agreement_n = submitter_agreement_n
         self._society_assigned_agreement_n = society_assigned_agreement_n
 
+        # Publisher chain link
+        self._publisher_sequence_n = publisher_sequence_n
+
     @property
     def publisher_ip_n(self):
         """
@@ -1082,6 +1086,21 @@ def publisher_name(self):
     def publisher_name(self, value):
         self._publisher_name = value
 
+    @property
+    def publisher_sequence_n(self):
+        """
+        Publisher Sequence # field. Numeric.
+
+        Must match the Publisher Sequence # of the relating SPU/OPU record.
+
+        :return: the publisher sequence number in the chain
+        """
+        return self._publisher_sequence_n
+
+    @publisher_sequence_n.setter
+    def publisher_sequence_n(self, value):
+        self._publisher_sequence_n = value
+
 
 class WriterRecord(InterestedPartyRecord):
     """
@@ -1100,9 +1119,9 @@ def __init__(self,
                  writer=None,
                  writer_designation=None,
                  work_for_hire=False,
-                 writer_unknown='F',
-                 reversionary='U',
-                 first_recording_refusal='U',
+                 writer_unknown='',
+                 reversionary='',
+                 first_recording_refusal='N',
                  usa_license='',
                  pr_society=None,
                  pr_ownership_share=0,

cwr/parser/decoder/dictionary.py

@@ -587,6 +587,8 @@ def decode(self, data):
                                     edi_standard=data['edi_standard'])
         if 'character_set' in data:
             header.character_set = data['character_set']
+        if 'version_type' in data:
+            header.version_type = data['version_type']
 
         return header
 

cwr/parser/encoder/dictionary.py

@@ -627,6 +627,7 @@ def encode(self, record):
             record.society_assigned_agreement_n
         encoded['submitter_agreement_n'] = record.submitter_agreement_n
         encoded['writer_ip_n'] = record.writer_ip_n
+        encoded['publisher_sequence_n'] = record.publisher_sequence_n
 
         return encoded
 
@@ -924,6 +925,12 @@ def encode(self, record):
         encoded['media_type'] = record.media_type
         encoded['recording_format'] = record.recording_format
         encoded['recording_technique'] = record.recording_technique
+        encoded['recording_title'] = getattr(record, 'recording_title', '')
+        encoded['version_title'] = getattr(record, 'version_title', '')
+        encoded['display_artist'] = getattr(record, 'display_artist', '')
+        encoded['record_label'] = getattr(record, 'record_label', '')
+        encoded['isrc_validity'] = getattr(record, 'isrc_validity', '')
+        encoded['submitter_recording_identifier'] = getattr(record, 'submitter_recording_identifier', '')
 
         return encoded
 
@@ -1097,6 +1104,7 @@ def encode(self, record):
         encoded['transmission_date'] = record.transmission_date
         encoded['edi_standard'] = record.edi_standard
         encoded['character_set'] = record.character_set
+        encoded['version_type'] = record.version_type
 
         return encoded
 

cwr/parser/encoder/standart/field.py

@@ -79,7 +79,7 @@ def format(self, value):
         if value is None:
             return self._blank()
         tpl = "{!s:<%d}" % self._rule['size']
-        return tpl.format(str(value))
+        return tpl.format(str(value))[:self._rule['size']]
 
 
 class NumericCwrFieldEncoder(CwrFieldEncoder):
@@ -90,7 +90,7 @@ def format(self, value):
         if value is None or value == '':
             value = 0
         tpl = "{:0>%d}" % self._rule['size']
-        return tpl.format(str(value))
+        return tpl.format(str(value))[:self._rule['size']]
 
 
 class NumericEmptyCwrFieldEncoder(CwrFieldEncoder):
@@ -102,7 +102,7 @@ def format(self, value):
             return self._blank()
         else:
             tpl = "{:0>%d}" % self._rule['size']
-            return tpl.format(str(value))
+            return tpl.format(str(value))[:self._rule['size']]
 
 
 class BooleanCwrFieldEncoder(DefaultCwrFieldEncoder):
@@ -170,9 +170,13 @@ def set_values(self, values):
 
 class LookupNumericCwrFieldEncoder(NumericEmptyCwrFieldEncoder, LookupCwrFieldEncoder):
     """
-    Lookup_int type encoder. Mu
+    Lookup_int type encoder for numeric table codes (e.g. society codes).
+    Treats 0 as blank since code 0 is not a valid lookup value.
     """
-    pass
+    def format(self, value):
+        if value == 0:
+            return self._blank()
+        return super(LookupNumericCwrFieldEncoder, self).format(value)
 
 class BlankCwrFieldEncoder(DefaultCwrFieldEncoder):
     """

cwr/parser/encoder/standart/record.py

@@ -192,6 +192,13 @@ def _process_record(rules):
                     processed[rule_id].append(rule)
                 else:
                     processed[rule_id] = [rule]
+        # Add aliases for equivalent territory-of-control record types:
+        # - Treat OWT as SWT for field templates
+        # - Treat OPT as SPT for field templates
+        if 'SWT' in processed and 'OWT' not in processed:
+            processed['OWT'] = processed['SWT']
+        if 'SPT' in processed and 'OPT' not in processed:
+            processed['OPT'] = processed['SPT']
         return processed
 
     def get_format_encoders(self, record_id):
@@ -201,7 +208,7 @@ def get_format_encoders(self, record_id):
 
     def get_encoder(self, entity):
         if entity.record_type not in self._record_configs:
-            raise NameError('The record type %s not found in config %s' % entity.record_type)
+            raise NameError('The record type %s not found in config %s' % (entity.record_type, list(self._record_configs.keys())))
         record_configs = self._record_configs[entity.record_type]
         if isinstance(entity, TransactionRecord):
             return TransactionCwrRecordEncoder(record_configs, self._field_configs)