Merge assets & JNI Native libraries from AAR

appinventor/appengine/src/com/google/appinventor/server/FileImporterImpl.java

@@ -52,7 +52,7 @@
 public final class FileImporterImpl implements FileImporter {
 
   // Maximum size of an uploaded asset, in megabytes.
-  private static final Flag<Float> maxAssetSizeMegs = Flag.createFlag("max.asset.size.megs", 9f);
+  private static final Flag<Float> maxAssetSizeMegs = Flag.createFlag("max.asset.size.megs", 15f);
 
   private static final Logger LOG = Logger.getLogger(FileImporterImpl.class.getName());
 

appinventor/appengine/src/com/google/appinventor/server/storage/ObjectifyStorageIo.java

@@ -1184,7 +1184,7 @@ public void run(Objectify datastore) {
   @Override
   public int getMaxJobSizeBytes() {
     // TODO(user): what should this mean?
-    return 5 * 1024 * 1024;
+    return 15 * 1024 * 1024;
   }
 
   @Override

appinventor/buildserver/src/com/google/appinventor/buildserver/tasks/android/AttachAarLibs.java

@@ -7,19 +7,31 @@
 
 import com.google.appinventor.buildserver.BuildType;
 import com.google.appinventor.buildserver.TaskResult;
+import com.google.appinventor.buildserver.YoungAndroidConstants;
 import com.google.appinventor.buildserver.context.AndroidCompilerContext;
+import com.google.appinventor.buildserver.context.AndroidPaths;
 import com.google.appinventor.buildserver.interfaces.AndroidTask;
 import com.google.appinventor.buildserver.util.AARLibraries;
 import com.google.appinventor.buildserver.util.AARLibrary;
 import com.google.appinventor.buildserver.util.ExecutorUtils;
 
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
-
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
+import java.util.Enumeration;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Set;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+import static com.google.appinventor.common.constants.YoungAndroidStructureConstants.ASSETS_FOLDER;
 
 
 /**
@@ -31,32 +43,69 @@ public class AttachAarLibs implements AndroidTask {
   @Override
   public TaskResult execute(AndroidCompilerContext context) {
     final File explodedBaseDir = ExecutorUtils.createDir(context.getPaths().getBuildDir(),
-        "exploded-aars");
+            "exploded-aars");
     final File generatedDir = ExecutorUtils.createDir(context.getPaths().getBuildDir(),
-        "generated");
+            "generated");
     final File genSrcDir = ExecutorUtils.createDir(generatedDir, "src");
     context.getComponentInfo().setExplodedAarLibs(new AARLibraries(genSrcDir));
     final Set<String> processedLibs = new HashSet<>();
 
     // Attach the Android support libraries (needed by every app)
     context.getComponentInfo().getLibsNeeded().put("ANDROID", new HashSet<>(Arrays.asList(
-        context.getResources().getSupportAars())));
+            context.getResources().getSupportAars())));
+
+    // Gather AAR assets to be added to apk's Asset directory.
+    // The assets directory have been created before this.
+    File mergedAssetDir = ExecutorUtils.createDir(context.getProject().getBuildDirectory(),
+            ASSETS_FOLDER);
+
+    // Root directory for JNI native (.so) libraries
+    AndroidPaths paths = context.getPaths();
+    File libsDir = ExecutorUtils.createDir(paths.getBuildDir(), YoungAndroidConstants.LIBS_DIR_NAME);
+    paths.setLibsDir(libsDir);
 
     // walk components list for libraries ending in ".aar"
     try {
-      for (Set<String> libs : context.getComponentInfo().getLibsNeeded().values()) {
-        Iterator<String> i = libs.iterator();
+      final HashSet<String> attachedAARs = new HashSet<>();
+      for (String type : context.getComponentInfo().getLibsNeeded().keySet()) {
+        Iterator<String> i = context.getComponentInfo().getLibsNeeded().get(type).iterator();
         while (i.hasNext()) {
           String libname = i.next();
+          String sourcePath = "";
           if (libname.endsWith(".aar")) {
             i.remove();
             if (!processedLibs.contains(libname)) {
-              // explode libraries into ${buildDir}/exploded-aars/<package>/
-              AARLibrary aarLib = new AARLibrary(new File(context.getResource(
-                  context.getResources().getRuntimeFilesDir() + libname)));
-              aarLib.unpackToDirectory(explodedBaseDir);
-              context.getComponentInfo().getExplodedAarLibs().add(aarLib);
-              processedLibs.add(libname);
+              if (context.getSimpleCompTypes().contains(type) || "ANDROID".equals(type)) {
+                final String pathSuffix = context.getResources().getRuntimeFilesDir() + libname;
+                sourcePath = context.getResource(pathSuffix);
+              } else if (context.getExtCompTypes().contains(type)) {
+                final String pathSuffix = "/aars/" + libname;
+                sourcePath = ExecutorUtils.getExtCompDirPath(type, context.getProject(),
+                        context.getExtTypePathCache()) + pathSuffix;
+              } else {
+                context.getReporter().error("Unknown component type: " + type, true);
+                return TaskResult.generateError("Error while attaching AAR libraries");
+              }
+
+              File aarFile = new File(sourcePath);
+              // Resolve possible conflicts
+              final String packageName = getAarPackageName(aarFile);
+              if (packageName == null || packageName.trim().isEmpty()) {
+                context.getReporter().error("Unable to read packageName from: " + aarFile.getName(), true);
+                return TaskResult.generateError("Unable to read packageName from: " + aarFile.getName());
+              }
+              if (!attachedAARs.contains(packageName)) {
+                // explode libraries into ${buildDir}/exploded-aars/<package>/
+                AARLibrary aarLib = new AARLibrary(aarFile);
+                aarLib.unpackToDirectory(explodedBaseDir);
+                context.getComponentInfo().getExplodedAarLibs().add(aarLib);
+                // Attach assets files & jni libraries if available
+                copyAssetsAndJni(aarFile, mergedAssetDir, libsDir);
+                processedLibs.add(libname);
+                attachedAARs.add(packageName);
+              } else {
+                System.out.println("Skip attaching duplicate AAR: " + aarFile.getName());
+              }
             }
           }
         }
@@ -68,4 +117,95 @@ public TaskResult execute(AndroidCompilerContext context) {
 
     return TaskResult.generateSuccess();
   }
+
+  private void copyAssetsAndJni(File aarFile, File mergedAssetDir, File libsDir) throws IOException {
+    try (ZipFile zip = new ZipFile(aarFile)) {
+      Enumeration<? extends ZipEntry> entries = zip.entries();
+
+      final Set<String> supportedABIs = new HashSet<>();
+      supportedABIs.add("arm64-v8a");
+      supportedABIs.add("armeabi-v7a");
+      supportedABIs.add("x86");
+      supportedABIs.add("x86_64");
+
+      while (entries.hasMoreElements()) {
+        ZipEntry entry = entries.nextElement();
+        File targetFile = null;
+        final String entryName = entry.getName();
+
+        if (entryName.startsWith("assets/")) {
+          if (!entry.isDirectory()) {
+            targetFile = new File(mergedAssetDir, entryName.substring("assets/".length()));
+            if (!targetFile.getParentFile().exists()) {
+              // The target file may contain subfolders.
+              targetFile.getParentFile().mkdirs();
+            }
+          }
+        } else if (entryName.startsWith("jni/") && entryName.endsWith(".so")) {
+          final String[] array = entryName.split("/", 3);
+          if (array.length < 3) {
+            continue;
+          }
+          final String abi = array[1];
+          if (!supportedABIs.contains(abi)) {
+            System.out.println("Skip merging not supported ABI: " + abi);
+            continue;
+          }
+          final String libName = array[2];
+          if (!entry.isDirectory()) {
+            File parentFile = new File(libsDir, abi);
+            if (!parentFile.exists()) {
+              // Create the parent ABI directory if absent
+              parentFile.mkdir();
+            }
+            targetFile = new File(parentFile, libName);
+          }
+        }
+        if (targetFile != null && !targetFile.exists()) {
+          // Copy file contents from ZIP entry
+          try (InputStream is = zip.getInputStream(entry);
+               OutputStream os = new FileOutputStream(targetFile)) {
+            byte[] buffer = new byte[8192];
+            int bytesRead;
+            while ((bytesRead = is.read(buffer)) != -1) {
+              os.write(buffer, 0, bytesRead);
+            }
+          }
+        }
+      }
+    }
+  }
+
+  private String getAarPackageName(File aarFile) throws IOException {
+    try (ZipFile zip = new ZipFile(aarFile)) {
+      ZipEntry manifestEntry = zip.getEntry("AndroidManifest.xml");
+      if (manifestEntry == null) {
+        Enumeration<? extends ZipEntry> entries = zip.entries();
+        while (entries.hasMoreElements()) {
+          ZipEntry e = entries.nextElement();
+          if (!e.isDirectory() && e.getName().endsWith("AndroidManifest.xml")) {
+            manifestEntry = e;
+            break;
+          }
+        }
+      }
+      if (manifestEntry == null) {
+        return null; // No manifest found
+      }
+
+      try (InputStream in = zip.getInputStream(manifestEntry);
+           BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
+        String line;
+        while ((line = reader.readLine()) != null) {
+          if (line.contains("package=\"")) {
+            int start = line.indexOf("package=\"") + "package=\"".length();
+            int end = line.indexOf("\"", start);
+            return line.substring(start, end);
+          }
+        }
+      }
+
+    }
+    return null;
+  }
 }

appinventor/components/src/com/google/appinventor/components/scripts/ExternalComponentGenerator.java

@@ -12,8 +12,13 @@
 import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Paths;
-import java.util.*;
 import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
 import org.json.JSONException;
 import org.json.JSONArray;
 import org.json.JSONObject;
@@ -169,14 +174,22 @@ private static void generateExternalComponentBuildFiles(String packageName, List
       JSONObject componentBuildInfo = info.buildInfo;
       try {
         JSONArray librariesNeeded = componentBuildInfo.getJSONArray("libraries");
+        JSONArray librariesAar = new JSONArray();
+        boolean ensureFreshDir = true;
+
         for (int j = 0; j < librariesNeeded.length(); ++j) {
           // Copy Library files for Unjar and Jaring
           String library = librariesNeeded.getString(j);
           copyFile(buildServerClassDirPath + File.separator + library,
               extensionTempDirPath + File.separator + library);
+          if (library.endsWith(".aar")) {
+            copyExternalAar(library, packageName, ensureFreshDir);
+            librariesAar.put(library);
+            ensureFreshDir = false;
+          }
         }
         //empty the libraries meta-data to avoid redundancy
-        componentBuildInfo.put("libraries", new JSONArray());
+        componentBuildInfo.put("libraries", librariesAar);
       } catch(JSONException e) {
         // bad
         throw new IllegalStateException("Unexpected JSON exception parsing simple_components.json",
@@ -344,6 +357,25 @@ private static Boolean copyFile(String srcPath, String dstPath) {
     return true;
   }
 
+  private static void copyExternalAar(String library, String packageName, boolean ensureFreshDir)
+      throws IOException {
+    File sourceDir = new File(buildServerClassDirPath + File.separator);
+    File aarFile = new File(sourceDir, library);
+    if (!aarFile.exists() || !library.endsWith(".aar")) {
+      return;
+    }
+    // Get aar dest directory
+    File destDir = new File(externalComponentsDirPath + File.separator + packageName + File.separator);
+    File aarDestDir = new File(destDir, "aars");
+    if (ensureFreshDir) {
+      // Ensure fresh directory before put the first library
+      ensureFreshDirectory(aarDestDir.getPath(), "Unable to delete the aars directory for the extension.");
+    }
+
+    System.out.println("Extensions : " + "Copying file aar " + library);
+    copyFile(aarFile.getAbsolutePath(), aarDestDir.getAbsolutePath() + File.separator + library);
+  }
+
   /**
    * Copy a compiled classes related to a given extension in his package folder.
    *