diff --git a/pivy-importer/src/main/groovy/com/linkedin/python/importer/deps/DependencyDownloader.groovy b/pivy-importer/src/main/groovy/com/linkedin/python/importer/deps/DependencyDownloader.groovy
index 41727c54..312a868f 100644
--- a/pivy-importer/src/main/groovy/com/linkedin/python/importer/deps/DependencyDownloader.groovy
+++ b/pivy-importer/src/main/groovy/com/linkedin/python/importer/deps/DependencyDownloader.groovy
@@ -18,7 +18,6 @@ package com.linkedin.python.importer.deps
import com.linkedin.python.importer.pypi.PypiApiCache
import com.linkedin.python.importer.util.ProxyDetector
import groovy.util.logging.Slf4j
-import org.apache.commons.io.FilenameUtils
import org.apache.http.client.fluent.Request
@Slf4j
@@ -58,10 +57,7 @@ abstract class DependencyDownloader {
abstract downloadDependency(
String dep, boolean latestVersions, boolean allowPreReleases, boolean fetchExtras, boolean lenient)
- protected static File downloadArtifact(File destDir, String url) {
-
- def filename = FilenameUtils.getName(new URL(url).getPath())
- def contents = new File(destDir, filename)
+ protected static File downloadArtifact(File contents, String url) {
if (!contents.exists()) {
def proxy = ProxyDetector.maybeGetHttpProxy()
@@ -88,12 +84,27 @@ abstract class DependencyDownloader {
}
/**
- * Get the actual module name from artifact name, which has the correct letter case.
+ * Get the module name from artifact name, which has the correct letter case.
* @param filename the filename of artifact
* @param revision module version
- * @return actual module name, which is from PyPI
+ * @return module name
*/
- static String getActualModuleNameFromFilename(String filename, String revision) {
+ static String getModuleNameFromFilename(String filename, String revision) {
return filename.substring(0, filename.indexOf(revision) - 1)
}
+
+ /**
+ * Compose filename from artifact file name and module name.
+ * Makes filename that has to be compatible with ivy layout pattern [module]-[revision].[ext],
+ * where [module] is from PyPI metadata.
+ * @param name module name from PyPI metadata
+ * @param filename the filename of artifact
+ * @param revision module version
+ * @return
+ */
+ static String buildFilenameByModuleName(String moduleName, String filename, String revision) {
+ final String moduleNameFromFilename = getModuleNameFromFilename(filename, revision)
+
+ return filename.replace(moduleNameFromFilename, moduleName)
+ }
}
diff --git a/pivy-importer/src/main/groovy/com/linkedin/python/importer/deps/SdistDownloader.groovy b/pivy-importer/src/main/groovy/com/linkedin/python/importer/deps/SdistDownloader.groovy
index da2374b8..1a9dfa96 100644
--- a/pivy-importer/src/main/groovy/com/linkedin/python/importer/deps/SdistDownloader.groovy
+++ b/pivy-importer/src/main/groovy/com/linkedin/python/importer/deps/SdistDownloader.groovy
@@ -18,6 +18,7 @@ package com.linkedin.python.importer.deps
import com.linkedin.python.importer.distribution.SourceDistPackage
import com.linkedin.python.importer.ivy.IvyFileWriter
import groovy.util.logging.Slf4j
+
import java.nio.file.Paths
@Slf4j
@@ -58,13 +59,16 @@ class SdistDownloader extends DependencyDownloader {
}
// make sure the module name has the right letter case and dash or underscore as PyPI
- name = getActualModuleNameFromFilename(sdistDetails.filename, version)
+ name = projectDetails.name
log.info("Pulling in $name:$version")
def destDir = Paths.get(ivyRepoRoot.absolutePath, SOURCE_DIST_ORG, name, version).toFile()
destDir.mkdirs()
- def sdistArtifact = downloadArtifact(destDir, sdistDetails.url)
+ def filename = buildFilenameByModuleName(name, sdistDetails.filename, version)
+ def contents = new File(destDir, filename)
+
+ def sdistArtifact = downloadArtifact(contents, sdistDetails.url)
def packageDependencies = new SourceDistPackage(name, version, sdistArtifact, cache, dependencySubstitution)
.getDependencies(latestVersions, allowPreReleases, fetchExtras, lenient)
diff --git a/pivy-importer/src/main/groovy/com/linkedin/python/importer/deps/WheelsDownloader.groovy b/pivy-importer/src/main/groovy/com/linkedin/python/importer/deps/WheelsDownloader.groovy
index 064a2fbe..f545ce9d 100644
--- a/pivy-importer/src/main/groovy/com/linkedin/python/importer/deps/WheelsDownloader.groovy
+++ b/pivy-importer/src/main/groovy/com/linkedin/python/importer/deps/WheelsDownloader.groovy
@@ -73,13 +73,16 @@ class WheelsDownloader extends DependencyDownloader {
}
// make sure the module name has the same letter case as PyPI
- name = getActualModuleNameFromFilename(wheelDetails.filename, version)
+ name = projectDetails.name
log.info("Pulling in $name:$version:$classifier")
def destDir = Paths.get(ivyRepoRoot.absolutePath, BINARY_DIST_ORG, name, version, classifier).toFile()
destDir.mkdirs()
- def wheelArtifact = downloadArtifact(destDir, wheelDetails.url)
+ def filename = buildFilenameByModuleName(name, wheelDetails.filename, version)
+ def contents = new File(destDir, filename)
+
+ def wheelArtifact = downloadArtifact(contents, wheelDetails.url)
def packageDependencies = new WheelsPackage(name, version, wheelArtifact, cache, dependencySubstitution)
.getDependencies(latestVersions, allowPreReleases, fetchExtras, lenient)
diff --git a/pivy-importer/src/test/groovy/com/linkedin/python/importer/deps/DependencyDownloaderTest.groovy b/pivy-importer/src/test/groovy/com/linkedin/python/importer/deps/DependencyDownloaderTest.groovy
index 8fb94b6e..e4c7d42d 100644
--- a/pivy-importer/src/test/groovy/com/linkedin/python/importer/deps/DependencyDownloaderTest.groovy
+++ b/pivy-importer/src/test/groovy/com/linkedin/python/importer/deps/DependencyDownloaderTest.groovy
@@ -80,7 +80,7 @@ class DependencyDownloaderTest extends Specification {
assert dependencyDownloader.processedDependencies.contains(testProject)
}
- def "get actual module name from filename"() {
+ def "get module name from filename"() {
given:
String testSdistFilename
String actualModuleName
@@ -88,31 +88,80 @@ class DependencyDownloaderTest extends Specification {
when:
testSdistFilename = "zc.buildout-2.12.1.tar.gz"
- actualModuleName = DependencyDownloader.getActualModuleNameFromFilename(testSdistFilename, "2.12.1")
+ actualModuleName = DependencyDownloader.getModuleNameFromFilename(testSdistFilename, "2.12.1")
expectedModuleName = "zc.buildout"
then:
actualModuleName == expectedModuleName
when:
testSdistFilename = "google-api-python-client-1.7.3.tar.gz"
- actualModuleName = DependencyDownloader.getActualModuleNameFromFilename(testSdistFilename, "1.7.3")
+ actualModuleName = DependencyDownloader.getModuleNameFromFilename(testSdistFilename, "1.7.3")
expectedModuleName = "google-api-python-client"
then:
actualModuleName == expectedModuleName
when:
testSdistFilename = "sphinx_rtd_theme-0.4.0-py2.py3-none-any.whl"
- actualModuleName = DependencyDownloader.getActualModuleNameFromFilename(testSdistFilename, "0.4.0")
+ actualModuleName = DependencyDownloader.getModuleNameFromFilename(testSdistFilename, "0.4.0")
expectedModuleName = "sphinx_rtd_theme"
then:
actualModuleName == expectedModuleName
when:
testSdistFilename = "psycopg2_binary-2.7.5-cp34-cp34m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl"
- actualModuleName = DependencyDownloader.getActualModuleNameFromFilename(testSdistFilename, "2.7.5")
+ actualModuleName = DependencyDownloader.getModuleNameFromFilename(testSdistFilename, "2.7.5")
expectedModuleName = "psycopg2_binary"
then:
assert actualModuleName == expectedModuleName
}
+
+ def "build filename by module name"() {
+ given:
+ String testSdistFilename
+ String moduleName
+ String filename
+ String expectedFilename
+
+ when:
+ moduleName = "zc.buildout"
+ testSdistFilename = "zc.buildout-2.12.1.tar.gz"
+ filename = DependencyDownloader.buildFilenameByModuleName(moduleName, testSdistFilename, "2.12.1")
+ expectedFilename = "zc.buildout-2.12.1.tar.gz"
+ then:
+ filename == expectedFilename
+
+ when:
+ moduleName = "google-api-python-client"
+ testSdistFilename = "google-api-python-client-1.7.3.tar.gz"
+ filename = DependencyDownloader.buildFilenameByModuleName(moduleName, testSdistFilename, "1.7.3")
+ expectedFilename = "google-api-python-client-1.7.3.tar.gz"
+ then:
+ filename == expectedFilename
+
+ when:
+ moduleName = "sphinx-rtd-theme"
+ testSdistFilename = "sphinx_rtd_theme-0.4.0-py2.py3-none-any.whl"
+ filename = DependencyDownloader.buildFilenameByModuleName(moduleName, testSdistFilename, "0.4.0")
+ expectedFilename = "sphinx-rtd-theme-0.4.0-py2.py3-none-any.whl"
+ then:
+ filename == expectedFilename
+
+ when:
+ moduleName = "psycopg2-binary"
+ testSdistFilename = "psycopg2_binary-2.7.5-cp34-cp34m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl"
+ filename = DependencyDownloader.buildFilenameByModuleName(moduleName, testSdistFilename, "2.7.5")
+ expectedFilename = "psycopg2-binary-2.7.5-cp34-cp34m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl"
+ then:
+ filename == expectedFilename
+
+ when:
+ moduleName = "typed_ast"
+ testSdistFilename = "typed-ast-1.3.2.tar.gz"
+ filename = DependencyDownloader.buildFilenameByModuleName(moduleName, testSdistFilename, "1.3.2")
+ expectedFilename = "typed_ast-1.3.2.tar.gz"
+
+ then:
+ assert filename == expectedFilename
+ }
}