[gentoo-commits] proj/java-ebuilder:master commit in: scripts/, /, src/main/java/org/gentoo/java/ebuilder/maven/, ...

[Michał Górny]

commit:     96d9679d0a19f3bbaf404005465b924eb3018e08
Author:     heroxbd <heroxbd <AT> gmail <DOT> com>
AuthorDate: Wed Oct  5 15:31:59 2016 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Wed Oct  5 15:31:59 2016 +0000
URL:        https://gitweb.gentoo.org/proj/java-ebuilder.git/commit/?id=96d9679d

Major updates to resolve dependency (#8)

* MavenEbuilder: mavenProject targetversion is not an array.

* MavenVersion: expand regex.

 1. match 3.1.4.GA (org.jboss.logging:jboss-logging)
 2. match 2.0b6 (org.apache-extras.beanshell:bsh)

* MavenCache: artifactId may not be identical to gentoo package.

* MavenCache: give explicit error messages as place holders when
  dependency is missing.

* script/meta.sh: auxiliary script to relate gentoo repo to maven
  repo metadata.

* Specify the default jre version.

* tree.sh: recursively call java-ebuilder.

* meta.sh: use maven to enumerate child modules.

* meta.sh: add more exceptions.

* MavenEbuilder.java: handle SLOT dependency according to ebuild styles.

  If SLOT=0, no SLOT is used in CLASSPATH.

* MavenEbuilder.java: remove JAVA_SRC_DIR.

  Not needed.  The default is enough.
  Because we have one ebuild per child module.

* MavenParser.java: mvn timeout to 10minutes.

  When something cannot be downloaded, maven could use several minutes
  to finish.

* MavenParser.java: let scope default to "compile".

  It is interpreted as common dependencies.  Some poms does not
  specify scopes of their dependencies.

* MavenParser.java: remove maven build instructions.

  They are not used at all.  Parsing them can cause error in some poms.

  The side effect is that testdependencies are not triggered.

* MavenProject.java: remove system dependencies.

  They are deprecated:

    https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html#System_Dependencies

  They are satisfied by virtual/jdk.

* MavenProject.java: source encoding default to UTF-8.

* PortageParser.java: consider dev-java/ant-* ebuilds.

* MavenParser.java: parse opengl-api version from pom.

 README                                             |  23 +++++
 scripts/meta.sh                                    | 101 +++++++++++++++++++
 scripts/tree.sh                                    | 111 +++++++++++++++++++++
 .../org/gentoo/java/ebuilder/maven/MavenCache.java |   6 +-
 .../gentoo/java/ebuilder/maven/MavenEbuilder.java  |  23 +++--
 .../gentoo/java/ebuilder/maven/MavenParser.java    |  14 +--
 .../gentoo/java/ebuilder/maven/MavenProject.java   |   6 +-
 .../gentoo/java/ebuilder/portage/MavenVersion.java |   2 +-
 .../java/ebuilder/portage/PortageParser.java       |   6 ++
 9 files changed, 269 insertions(+), 23 deletions(-)

diff --git a/README b/README
new file mode 100644
index 0000000..e74f482
--- /dev/null
+++ b/README
@@ -0,0 +1,23 @@
+Java team tool for semi-automatic creation of ebuilds from pom.xml.
+
+Use example:
+
+1. Generate cache of java related packages from portage tree.
+java-ebuilder --refresh-cache -t /usr/portage
+
+2. Find corresponding maven groupId, artifactId and version in the cache.
+cd $HOME/.java-ebuilder
+mv cache cache.raw0
+( echo 1.0; tail -n +2 cache.raw0 | parallel -j -2 scripts/meta.sh; ) > cache.0
+
+3. Generate cache of java related packages from java overaly.
+java-ebuilder --refresh-cache -t /var/lib/layman/java
+
+4. Find corresponding maven groupId, artifactId and version in the cache.
+cd $HOME/.java-ebuilder
+mv cache cache.raw1
+( echo 1.0; tail -n +2 cache.raw1 | parallel -j -2 scripts/meta.sh; ) > cache.1
+
+5. Create ebuild recursively.
+script/tree.sh org.apache.spark:spark-core_2.11:2.0.0
+

diff --git a/scripts/meta.sh b/scripts/meta.sh
new file mode 100755
index 0000000..0a142c8
--- /dev/null
+++ b/scripts/meta.sh
@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+# read in cache from java-ebuilder and find out the groupId,
+# artifactId and version.
+
+# cache is by default at $HOME/.java-ebuilder/cache
+# example:
+# ( echo 1.0; tail -n +2 cache | parallel -j -2 meta.sh; ) > cache.1
+
+pkg=$(awk -F ":" '{print $1"/"$2"-"$3}' <<< $1)
+spkg=$(cut -d: -f2 <<< $1)
+sver=$(cut -d: -f3 <<< $1)
+case ${spkg} in
+    guava) 
+        echo $1:com.google.guava:${spkg}:${sver%%-*}
+        exit 0
+        ;;
+    json)
+        echo $1:org.json:${spkg}:${sver%%-*}
+        exit 0
+        ;;
+    xerces)
+        echo $1:xerces:${spkg}:${sver%%-*}
+        echo $1:xerces:xercesImpl:${sver%%-*}
+        echo $1:xerces:xmlParserAPIs:${sver%%-*}
+        exit 0
+        ;;
+    commons-*|classworlds|jdom)
+        echo $1:${spkg}:${spkg}:${sver%%-*}
+        exit 0
+        ;;
+    velocity)
+        echo $1:org.apache.${spkg}:${spkg}:${sver%%-*}
+        exit 0
+        ;;
+    jtidy)
+        echo $1:net.sf.${spkg}:${spkg}:${sver%%-*}
+        exit 0
+        ;;
+    saxpath)
+        echo $1:${spkg}:${spkg}:${sver%%-*}-FCS # maven version
+        echo $1:org.jdom:${spkg}:${sver%%-*}-FCS
+        exit 0
+        ;;
+    ant*)
+        [[ ${spkg} = ant-core ]] && { spkg=ant; echo $1:ant:${spkg}:${sver%%-*}; }
+        [[ ${spkg} = ant-ivy ]] && spkg=ivy
+        echo $1:org.apache.ant:${spkg}:${sver%%-*}
+        exit 0
+        ;;
+    bcpg|bcprov)
+        echo $1:org.bouncycastle:${spkg}-jdk12:130
+        for jv in 1{4,5,6}; do
+            echo $1:org.bouncycastle:${spkg}-jdk${jv}:${sver%%-*}
+        done
+        exit 0
+        ;;
+    asm)
+        echo $1:org.ow2.asm:${spkg}:${sver%%-*}
+        echo $1:org.ow2.asm:${spkg}-all:${sver%%-*}
+        echo $1:org.ow2.asm:${spkg}-debug-all:${sver%%-*}
+        exit 0
+esac
+
+grep -q ${pkg} <bebd <bpom && exit 0
+
+ebd=$(equery w ${pkg} 2>/dev/null)
+if [[ -z "${ebd}" ]]; then
+    echo $1:${pkg} >> bebd
+    exit 0
+fi
+
+# java-utils-2.eclass:java-pkg_needs-vm()
+export JAVA_PKG_NV_DEPEND="nothing"
+
+if ! ebuild ${ebd} unpack >/dev/null 2>&1; then
+    echo $1:${pkg} >> bebd
+    exit 0
+fi
+
+bad_pom="yes"
+for subd in /dev/shm/portage/${pkg}/work/*; do
+    [[ -f ${subd}/pom.xml ]] || continue
+    bad_pom=""
+    pushd ${subd} > /dev/null
+    poms=$(mvn -q --also-make exec:exec -Dexec.executable="pwd" 2> /dev/null | grep ^/)
+    popd > /dev/null
+    for pd in ${poms}; do
+        ppom=$(xml2 < ${pd}/pom.xml | egrep '(groupId|artifactId|version)=')
+        PG=$(echo "${ppom}" | sed -n -r -e 's,/project/groupId=(.*),\1,p')
+        [[ -z ${PG} ]] && PG=$(echo "${ppom}" | sed -n -r -e 's,/project/parent/groupId=(.*),\1,p')
+        PA=$(echo "${ppom}" | sed -n -r -e 's,/project/artifactId=(.*),\1,p')
+        PV=$(echo "${ppom}" | sed -n -r -e 's,/project/version=(.*),\1,p')
+        [[ -z ${PV} ]] && PV=$(echo "${ppom}" | sed -n -r -e 's,/project/parent/version=(.*),\1,p')
+        echo $1:${PG}:${PA}:${PV/-SNAPSHOT/}
+    done
+done
+if [[ -n "${bad_pom}" ]]; then
+    echo $1:${pkg} >> bpom
+fi
+
+ebuild ${ebd} clean >/dev/null 2>&1

diff --git a/scripts/tree.sh b/scripts/tree.sh
new file mode 100755
index 0000000..5482a0b
--- /dev/null
+++ b/scripts/tree.sh
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+# start from the root of a maven artifact and recursively resolve its
+# dependencies.
+
+mkdir -p ../poms
+
+gebd() {
+    case ${MA} in
+        weld-osgi-bundle)
+            # 1.1.0.Final no longer exist
+            [[ ${MV} = 1.1.0.Final ]] && MV=1.1.33.Final
+            ;;
+    esac
+
+    local WORKDIR=${PG//./\/}/${MA} MID
+    local MID=${PG}:${MA}:${MV}
+    # .Final .GA .v20121024 means nothing
+    local PV=${MV%.[a-zA-Z]*} PA SLOT
+
+    case ${MA} in
+        opengl-api)
+            [[ ${MV} = 2.1.1 ]] && MV=gl1.1-android-2.1_r1
+            ;;
+    esac
+
+    # plexus-container-default 1.0-alpha-9-stable-1
+    PV=${PV/-stable-*/}
+    PV=${PV/-alpha-/_alpha}
+    # wagon-provider-api 1.0-beta-7
+    PV=${PV/-beta-/_beta}
+    # aopalliance-repackaged 2.5.0-b16
+    PV=${PV/-b/_beta}
+    # javax.xml.stream:stax-api:1.0-2
+    PV=${PV//-/.}
+
+    local M=${MA}-${MV}
+    local SRC_URI="http://central.maven.org/maven2/${WORKDIR}/${MV}/${M}-sources.jar"
+
+    # spark-launcher_2.11 for scala 2.11
+    eval $(sed -nr 's,([^_]*)(_(.*))?,PA=\1 SLOT=\3,p' <<< ${MA})
+    [[ -z "${SLOT}" ]] && eval $(sed -nr 's,(.*)-(([0-9]+\.)?[0-9]+),PA=\1 SLOT=\2,p' <<< ${MA})
+    [[ -z "${SLOT}" ]] && PA=${MA}
+    PA=${PA//./-}
+    PA=${PA//_/-}
+    local P=${PA}-${PV}
+    local ebd=app-maven/${PA}/${P}.ebuild
+
+    if [[ ! -f ../poms/${M}.pom ]]; then
+        pushd ../poms
+        wget ${SRC_URI/-sources.jar/.pom}
+
+        # 3rd party plugin not needed here
+        # distributionManagement is invalid for maven 3
+        # net.sf.jtidy:jtidy:r938 version is not maven-compliant
+        sed -e '/<packaging>bundle/d' \
+            -e '/<distributionManagement>/,/<\/distributionManagement>/d' \
+            -e '/<build>/,/<\/build>/d' \
+            -e '/<modules>/,/<\/modules>/d' \
+            -e 's,<version>r938</version>,<version>1.0</version>,' \
+            -i ${M}.pom
+        popd
+    fi
+
+    wget -q --spider ${SRC_URI} || SRC_URI=${SRC_URI/-sources.jar/.jar}
+
+    if [[ ! -f app-maven/${PA}/${P}.ebuild ]]; then
+        mkdir -p app-maven/${PA}
+        java-ebuilder -p ../poms/${M}.pom -e ${ebd} -g  --workdir . \
+                      -u ${SRC_URI} --slot ${SLOT:-0} --keywords ~amd64
+
+        # empty parent artifacts
+        # FIXME, this should be removed in poms
+        sed -i '/app-maven\/jsch-agentproxy-[0-9]/d' ${ebd}
+    fi
+
+    line=app-maven:${PA}:${PV}:${SLOT:-0}::${MID}
+    if ! grep -q ${line} ${HOME}/.java-ebuilder/maven-cache ; then
+        pushd ${HOME}/.java-ebuilder > /dev/null
+        echo ${line} >> maven-cache
+        cat cache.{0,1} maven-cache > cache
+        popd > /dev/null
+    fi
+
+    if [[ -z "${MAVEN_NODEP}" ]] && mfill app-maven/${PA}/${P}.ebuild; then
+        java-ebuilder -p ../poms/${M}.pom -e ${ebd} -g  --workdir . \
+                      -u ${SRC_URI} --slot ${SLOT:-0} --keywords ~amd64
+    fi
+
+    [[ ${SRC_URI} = *-sources.jar ]] || sed -i "/inherit/s/java-pkg-simple/java-pkg-binjar/" ${ebd}
+}
+
+mfill() {
+    # recursively fill missing dependencies
+    arts=$(sed -n -r 's,# (test\? )?(.*)-> !!!.*-not-found!!!,\2,p' < $1)
+    if [[ -z "${arts}" ]]; then
+        false # no need to java-ebuilder again
+    else
+        for a in ${arts}; do
+            eval $(awk -F":" '{print "PG="$1, "MA="$2, "MV="$3}' <<< ${a})
+            gebd
+        done
+        return
+    fi
+}
+
+if [[ $1 == *.ebuild ]]; then
+    mfill $1
+else
+    eval $(awk -F":" '{print "PG="$1, "MA="$2, "MV="$3}' <<< $1)
+    gebd
+fi

diff --git a/src/main/java/org/gentoo/java/ebuilder/maven/MavenCache.java b/src/main/java/org/gentoo/java/ebuilder/maven/MavenCache.java
index 1c139c1..1295856 100644
--- a/src/main/java/org/gentoo/java/ebuilder/maven/MavenCache.java
+++ b/src/main/java/org/gentoo/java/ebuilder/maven/MavenCache.java
@@ -43,13 +43,13 @@ public class MavenCache {
         final Map<String, List<CacheItem>> artifactIds = cache.get(groupId);
 
         if (artifactIds == null) {
-            return null;
+            return "!!!groupId-not-found!!!";
         }
 
         final List<CacheItem> versions = artifactIds.get(artifactId);
 
         if (versions == null) {
-            return null;
+            return "!!!artifactId-not-found!!!";
         }
 
         final MavenVersion mavenVersion = new MavenVersion(version);
@@ -156,7 +156,7 @@ public class MavenCache {
 
         if (versions == null) {
             versions = new ArrayList<>(10);
-            artifactIds.put(cacheItem.getPkg(), versions);
+            artifactIds.put(cacheItem.getArtifactId(), versions);
         }
 
         versions.add(cacheItem);

diff --git a/src/main/java/org/gentoo/java/ebuilder/maven/MavenEbuilder.java b/src/main/java/org/gentoo/java/ebuilder/maven/MavenEbuilder.java
index c6e18bd..067ea80 100644
--- a/src/main/java/org/gentoo/java/ebuilder/maven/MavenEbuilder.java
+++ b/src/main/java/org/gentoo/java/ebuilder/maven/MavenEbuilder.java
@@ -72,10 +72,18 @@ public class MavenEbuilder {
                         sbCP.append(',');
                     }
 
-                    sbCP.append(dependency.getSystemDependency().
-                            replaceAll(".*/", "").
-                            replaceAll("\\[.*\\]", "").
-                            replace(":", "-"));
+		    String[] parts = dependency.getSystemDependency().
+			replaceAll(".*/", "").
+			replaceAll("\\[.*\\]", "").
+			split(":");
+		    String PN = parts[0].replaceAll("-r\\d+$", "");
+		    if (parts.length == 2) {
+			PN = PN.substring(0, PN.lastIndexOf('-'));
+			if (!parts[1].equals("0")) {
+			    PN += "-" + parts[1];
+			}
+		    }
+                    sbCP.append(PN);
                 });
 
         return sbCP.toString();
@@ -301,7 +309,7 @@ public class MavenEbuilder {
                             mavenProject.getCommonDependencies(), null);
                 }
 
-                if (!mavenProject.getTargetVersion().isEmpty()) {
+                if (mavenProject.getTargetVersion()!=null) {
                     writeDependenciesInfo(writer, mavenProject.getPomFile(),
                             mavenProject.getTestDependencies(), "test?");
                 }
@@ -633,11 +641,6 @@ public class MavenEbuilder {
             writer.println('"');
         }
 
-        writer.print("JAVA_SRC_DIR=\"");
-        writer.print(config.getWorkdir().relativize(
-                mavenProject.getSourceDirectory()));
-        writer.println('"');
-
         if (mavenProject.hasResources()) {
             writer.print("JAVA_RESOURCE_DIRS=\"");
 

diff --git a/src/main/java/org/gentoo/java/ebuilder/maven/MavenParser.java b/src/main/java/org/gentoo/java/ebuilder/maven/MavenParser.java
index 52449dc..ba876f2 100644
--- a/src/main/java/org/gentoo/java/ebuilder/maven/MavenParser.java
+++ b/src/main/java/org/gentoo/java/ebuilder/maven/MavenParser.java
@@ -112,10 +112,10 @@ public class MavenParser {
         }
 
         try {
-            process.waitFor(1, TimeUnit.MINUTES);
+            process.waitFor(10, TimeUnit.MINUTES);
         } catch (final InterruptedException ex) {
             config.getErrorWriter().println("ERROR: mvn process did not finish "
-                    + "within 1 minute, exiting.");
+                    + "within 10 minute, exiting.");
             Runtime.getRuntime().exit(1);
         }
 
@@ -389,9 +389,6 @@ public class MavenParser {
                     case "artifactId":
                         mavenProject.setArtifactId(reader.getElementText());
                         break;
-                    case "build":
-                        parseProjectBuild(mavenProject, reader);
-                        break;
                     case "dependencies":
                         parseProjectDependencies(mavenProject, mavenCache,
                                 reader);
@@ -509,7 +506,7 @@ public class MavenParser {
         String groupId = null;
         String artifactId = null;
         String version = null;
-        String scope = null;
+        String scope = "compile";
 
         while (reader.hasNext()) {
             reader.next();
@@ -528,6 +525,11 @@ public class MavenParser {
                     case "version":
                         version = reader.getElementText().replace("-SNAPSHOT",
                                 "");
+			/* crazy version from
+			 * org.khronos:opengl-api:gl1.1-android-2.1_r1 */
+			if (version.equals("gl1.1-android-2.1_r1")) {
+			    version = "2.1.1";
+			}
                         break;
                     default:
                         consumeElement(reader);

diff --git a/src/main/java/org/gentoo/java/ebuilder/maven/MavenProject.java b/src/main/java/org/gentoo/java/ebuilder/maven/MavenProject.java
index adec6a6..af05c7c 100644
--- a/src/main/java/org/gentoo/java/ebuilder/maven/MavenProject.java
+++ b/src/main/java/org/gentoo/java/ebuilder/maven/MavenProject.java
@@ -63,7 +63,7 @@ public class MavenProject {
     /**
      * Source encoding.
      */
-    private String sourceEncoding = "1.7";
+    private String sourceEncoding = "UTF-8";
     /**
      * Source compile version.
      */
@@ -71,7 +71,7 @@ public class MavenProject {
     /**
      * Target compile version.
      */
-    private String targetVersion;
+    private String targetVersion = "1.7";
     /**
      * Test resource directories.
      */
@@ -155,7 +155,7 @@ public class MavenProject {
      * @return list of dependencies
      */
     public List<MavenDependency> getCompileDependencies() {
-        return getDependencies(new String[]{"provided", "system"});
+        return getDependencies(new String[]{"provided"});
     }
 
     /**

diff --git a/src/main/java/org/gentoo/java/ebuilder/portage/MavenVersion.java b/src/main/java/org/gentoo/java/ebuilder/portage/MavenVersion.java
index 002ea5a..e603943 100644
--- a/src/main/java/org/gentoo/java/ebuilder/portage/MavenVersion.java
+++ b/src/main/java/org/gentoo/java/ebuilder/portage/MavenVersion.java
@@ -14,7 +14,7 @@ public class MavenVersion implements Comparable<MavenVersion> {
      * Pattern for parsing maven version number.
      */
     private static final Pattern PATTERN_VERSION = Pattern.compile(
-            "^(\\d+)(?:\\.(\\d+))?(?:\\.(\\d+))?(?:-(.*))?$");
+            "^(\\d+)(?:\\.(\\d+))?(?:[\\.b](\\d+))?(?:[\\.-](.*))?$");
     /**
      * Incremental version number.
      */

diff --git a/src/main/java/org/gentoo/java/ebuilder/portage/PortageParser.java b/src/main/java/org/gentoo/java/ebuilder/portage/PortageParser.java
index c2cc976..aa8a14b 100644
--- a/src/main/java/org/gentoo/java/ebuilder/portage/PortageParser.java
+++ b/src/main/java/org/gentoo/java/ebuilder/portage/PortageParser.java
@@ -45,6 +45,10 @@ public class PortageParser {
      * Current java utilities eclass name.
      */
     private static final String ECLASS_JAVA_UTILS = "java-utils-2";
+    /**
+     * Current ant utilities eclass name.
+     */
+    private static final String ECLASS_ANT_TASKS = "ant-tasks";
     /**
      * Pattern for parsing SLOT with bash substring.
      */
@@ -126,6 +130,8 @@ public class PortageParser {
             return ECLASS_JAVA_PKG_SIMPLE;
         } else if (inheritLine.contains(ECLASS_JAVA_UTILS)) {
             return ECLASS_JAVA_UTILS;
+	} else if (inheritLine.contains(ECLASS_ANT_TASKS)) {
+	    return ECLASS_ANT_TASKS;
         } else {
             return null;
         }