WIP: track changes for oersi-etl

.github/workflows/publish.yml

@@ -0,0 +1,18 @@
+name: Publish package to GitHub Packages
+on:
+  push:
+    branches:
+      - '*-rc*'
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Publish package
+        run: ./gradlew publish
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+

build.gradle

@@ -47,7 +47,8 @@ editorconfig {
     '**/*.bzip2',
     '**/*.gzip',
     '**/*.xz',
-    'gradlew*'
+    'gradlew*',
+    '**/bin'
   ]
 }
 
@@ -56,6 +57,7 @@ subprojects {
   apply plugin: 'maven'
   apply plugin: 'jacoco'
   apply plugin: 'checkstyle'
+  apply plugin: 'maven-publish'
 
   check.dependsOn(editorconfigCheck)
   check.dependsOn(javadoc)
@@ -94,6 +96,7 @@ subprojects {
     options {
       addBooleanOption 'Xwerror', true
     }
+    exclude '**/harvester2/**'
   }
 
   artifacts {
@@ -241,9 +244,43 @@ gradle.projectsEvaluated {
       }
     }
 
+    //'upload' vs. 'publish, see
+    //https://medium.com/dot-debug/deploying-artifacts-to-maven-using-gradle-b669acc1b6f8'
+    /*
     task publish {
       dependsOn tasks.uploadArchives
     }
+    */
+
+    publishing {
+      publications {
+        mavenArtifacts(MavenPublication) {
+          from components.java
+          afterEvaluate {
+            groupId = project.group
+            artifactId = project.name
+          }
+        }
+      }
+
+      repositories {
+        maven {
+          name = 'localEmbedded'
+          // run generated gradle task `./gradlew
+          // publishMavenArtifactsPublicationToLocalEmbeddedRepository`
+          // to publish all subprojects into the same local embedded repo:
+          url = "file://${rootDir}/build/repo"
+        }
+        maven {
+          name = "GitHubPackages"
+          url = "https://maven.pkg.github.com/metafacture/metafacture-core"
+          credentials {
+            username = System.getenv("GITHUB_ACTOR")
+            password = System.getenv("GITHUB_TOKEN")
+          }
+        }
+      }
+    }
   }
 }
 
@@ -315,6 +352,10 @@ def getSnapshotVersion() {
     logger.lifecycle('Release branch found')
     return "${extractVersionFromBranch(grgit.branch.current().name)}-SNAPSHOT"
   }
+  if (grgit.branch.current().name.contains('-rc')) {
+    logger.lifecycle('Release candidate branch found')
+    return "${grgit.branch.current().name}"
+  }
   logger.lifecycle('Feature branch found')
   return "feature-${grgit.branch.current().name}-SNAPSHOT"
 }

config/checkstyle/checkstyle.xml

@@ -6,7 +6,7 @@
     <module name="SuppressWarningsFilter" />
     <module name="SuppressionSingleFilter">
         <property name="checks" value=".*"/>
-        <property name="files" value="generated-src"/>
+        <property name="files" value="generated-src|harvester2"/>
     </module>
     <module name="TreeWalker">
         <module name="AbstractClassName"/>

metafacture-biblio/build.gradle

@@ -21,13 +21,13 @@ dependencies {
   api project(':metafacture-framework')
   implementation project(':metafacture-commons')
   implementation project(':metafacture-flowcontrol')
-  implementation 'org.dspace:oclc-harvester2:0.1.12'
   implementation ('xalan:xalan:2.7.0') {
     exclude group: 'xalan', module: 'serializer'
     exclude group: 'xercesImpl', module: 'xercesImpl'
     exclude group: 'xml-apis', module: 'xml-apis'
   }
   implementation 'log4j:log4j:1.2.17'
+  implementation 'org.slf4j:slf4j-api:1.7.7'
   testImplementation 'junit:junit:4.12'
   testImplementation 'org.mockito:mockito-core:2.5.5'
 }

metafacture-biblio/src/main/java/org/metafacture/biblio/OaiPmhOpener.java

@@ -1,4 +1,4 @@
-/* Copyright 2013 Pascal Christoph.
+/* Copyright 2013, 2022 Pascal Christoph and others.
  * Licensed under the Eclipse Public License 1.0 */
 
 package org.metafacture.biblio;
@@ -10,7 +10,7 @@
 import org.metafacture.framework.annotations.Out;
 import org.metafacture.framework.helpers.DefaultObjectPipe;
 
-import ORG.oclc.oai.harvester2.app.RawWrite;
+import org.oclc.oai.harvester2.app.RawWrite;
 import org.xml.sax.SAXException;
 
 import java.io.ByteArrayInputStream;
@@ -19,7 +19,7 @@
 import java.io.InputStreamReader;
 import java.io.Reader;
 import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.TransformerException;
+import javax.xml.xpath.XPathException;
 
 /**
  * Opens an OAI-PMH stream and passes a reader to the receiver.
@@ -111,10 +111,10 @@ public void process(final String baseUrl) {
         catch (final SAXException e) {
             e.printStackTrace();
         }
-        catch (final TransformerException e) {
+        catch (final NoSuchFieldException e) {
             e.printStackTrace();
         }
-        catch (final NoSuchFieldException e) {
+        catch (final XPathException e) {
             e.printStackTrace();
         }
         try {

metafacture-biblio/src/main/java/org/oclc/oai/harvester2/app/RawWrite.java

@@ -0,0 +1,170 @@
+/**
+ * Copyright 2006 OCLC, Online Computer Library Center Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a copy of the
+ * License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is
+ * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and limitations under the License.
+ */
+package org.oclc.oai.harvester2.app;
+
+import java.io.*;
+import java.lang.NoSuchFieldException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.HashMap;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.xpath.XPathException;
+import javax.xml.xpath.XPathExpressionException;
+import org.oclc.oai.harvester2.verb.*;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+public class RawWrite {
+
+    public static void main(String[] args) {
+        try {
+            System.out.println(new Date());
+
+            HashMap<String, Object> options = getOptions(args);
+            List<?> rootArgs = (List<?>) options.get("rootArgs");
+            String baseURL = null;
+            if (rootArgs.size() > 0) {
+                baseURL = (String) rootArgs.get(0);
+            } else {
+                throw new IllegalArgumentException();
+            }
+
+            OutputStream out = System.out;
+            String outFileName = (String) options.get("-out");
+            String from = (String) options.get("-from");
+            String until = (String) options.get("-until");
+            String metadataPrefix = (String) options.get("-metadataPrefix");
+            if (metadataPrefix == null) metadataPrefix = "oai_dc";
+            String resumptionToken = (String) options.get("-resumptionToken");
+            String setSpec = (String) options.get("-setSpec");
+
+            if (resumptionToken != null) {
+                if (outFileName != null)
+                    out = new FileOutputStream(outFileName, true);
+                run(baseURL, resumptionToken, out);
+            } else {
+                if (outFileName != null)
+                    out = new FileOutputStream(outFileName);
+                run(baseURL, from, until, metadataPrefix, setSpec, out);
+            }
+
+            if (out != System.out) out.close();
+            System.out.println(new Date());
+        } catch (IllegalArgumentException e) {
+            System.err.println("RawWrite <-from date> <-until date> <-metadataPrefix prefix> <-setSpec setName> <-resumptionToken token> <-out fileName> baseURL");
+        } catch (Exception e) {
+            e.printStackTrace();
+            System.exit(-1);
+        }
+    }
+
+    public static void run(String baseURL, String resumptionToken,
+            OutputStream out)
+            throws IOException, ParserConfigurationException, SAXException, XPathExpressionException,
+            NoSuchFieldException {
+        ListRecords listRecords = new ListRecords(baseURL, resumptionToken);
+        while (listRecords != null) {
+            NodeList errors = listRecords.getErrors();
+            if (errors != null && errors.getLength() > 0) {
+                System.out.println("Found errors");
+                int length = errors.getLength();
+                for (int i = 0; i < length; ++i) {
+                    Node item = errors.item(i);
+                    System.out.println(item);
+                }
+                System.out.println("Error record: " + listRecords.toString());
+                break;
+            }
+//             System.out.println(listRecords);
+            out.write(listRecords.toString().getBytes("UTF-8"));
+            out.write("\n".getBytes("UTF-8"));
+            resumptionToken = listRecords.getResumptionToken();
+            System.out.println("resumptionToken: " + resumptionToken);
+            if (resumptionToken == null || resumptionToken.length() == 0) {
+                listRecords = null;
+            } else {
+                listRecords = new ListRecords(baseURL, resumptionToken);
+            }
+        }
+        out.write("</harvest>\n".getBytes("UTF-8"));
+    }
+
+    public static void run(String baseURL, String from, String until,
+            String metadataPrefix, String setSpec,
+            OutputStream out)
+            throws IOException, ParserConfigurationException, SAXException, XPathException,
+            NoSuchFieldException {
+        out.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n".getBytes("UTF-8"));
+        out.write("<harvest>\n".getBytes("UTF-8"));
+        out.write(new Identify(baseURL).toString().getBytes("UTF-8"));
+        out.write("\n".getBytes("UTF-8"));
+        out.write(new ListMetadataFormats(baseURL).toString().getBytes("UTF-8"));
+        out.write("\n".getBytes("UTF-8"));
+        ListSets listSets = new ListSets(baseURL);
+        while (listSets != null) {
+            out.write(listSets.toString().getBytes("UTF-8"));
+            out.write("\n".getBytes("UTF-8"));
+            String resumptionToken = listSets.getResumptionToken();
+            System.out.println("resumptionToken: " + resumptionToken);
+            if (resumptionToken == null || resumptionToken.length() == 0) {
+                listSets = null;
+            } else {
+                listSets = new ListSets(baseURL, resumptionToken);
+            }
+        }
+        ListRecords listRecords = new ListRecords(baseURL, from, until, setSpec,
+                metadataPrefix);
+        while (listRecords != null) {
+            NodeList errors = listRecords.getErrors();
+            if (errors != null && errors.getLength() > 0) {
+                System.out.println("Found errors");
+                int length = errors.getLength();
+                for (int i = 0; i < length; ++i) {
+                    Node item = errors.item(i);
+                    System.out.println(item);
+                }
+                System.out.println("Error record: " + listRecords.toString());
+                break;
+            }
+//             System.out.println(listRecords);
+            out.write(listRecords.toString().getBytes("UTF-8"));
+            out.write("\n".getBytes("UTF-8"));
+            String resumptionToken = listRecords.getResumptionToken();
+            System.out.println("resumptionToken: " + resumptionToken);
+            if (resumptionToken == null || resumptionToken.length() == 0) {
+                listRecords = null;
+            } else {
+                listRecords = new ListRecords(baseURL, resumptionToken);
+            }
+        }
+        out.write("</harvest>\n".getBytes("UTF-8"));
+    }
+
+    private static HashMap<String, Object> getOptions(String[] args) {
+        HashMap<String, Object> options = new HashMap<>();
+        ArrayList<String> rootArgs = new ArrayList<>();
+        options.put("rootArgs", rootArgs);
+
+        for (int i = 0; i < args.length; ++i) {
+            if (args[i].charAt(0) != '-') {
+                rootArgs.add(args[i]);
+            } else if (i + 1 < args.length) {
+                options.put(args[i], args[++i]);
+            } else {
+                throw new IllegalArgumentException();
+            }
+        }
+        return options;
+    }
+}