XIP Importer

Last modified by Clément Aubin on 2021/03/18 11:28

wrenchAllows to import XIP packages into the local XWiki Extension repository.
Type
Category
Developed by

Clément Aubin, Anca Luca, Thomas Mortagne

Rating
1 Votes
LicenseGNU Lesser General Public License 2.1

Table of contents

Description

In a new page, copy and paste the following content. Then, access the page.

{{groovy}}
 import org.apache.commons.fileupload.FileItem;
 import com.xpn.xwiki.plugin.fileupload.FileUploadPluginApi;
 import com.xpn.xwiki.XWikiContext;

 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;

 import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
 import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.input.CloseShieldInputStream;
 import org.apache.commons.lang3.StringUtils;
 import org.xwiki.environment.Environment;
 import org.xwiki.extension.Extension;
 import org.xwiki.extension.ExtensionId;
 import org.xwiki.extension.LocalExtension;
 import org.xwiki.extension.repository.InstalledExtensionRepository;
 import org.xwiki.extension.repository.LocalExtensionRepository;
 import org.xwiki.extension.repository.LocalExtensionRepositoryException;
 import org.xwiki.extension.repository.internal.ExtensionSerializer;
 import org.xwiki.extension.repository.internal.local.DefaultLocalExtension;

 class ExtensionEntry
  {
    DefaultLocalExtension descriptor;

    File file;

    boolean skip;
  }

 /**
  * Import the extension located in a XIP package in the local extensions repository.
  *
  * @param stream the stream containing the XIP file
  * @param selected the identifiers of the extensions to import or null if all extensions should be imported
  */
 def importXIP(InputStream stream, Set<ExtensionId> selected)
  {
    ExtensionSerializer extensionSerializer = services.component.getInstance(ExtensionSerializer.class);
    LocalExtensionRepository localRepository = services.component.getInstance(LocalExtensionRepository.class);
    InstalledExtensionRepository installedRepository = services.component.getInstance(InstalledExtensionRepository.class);
    Environment environment = services.component.getInstance(Environment.class);

    File tempFolder = new File(environment.getTemporaryDirectory(), "xipimporter/");
    tempFolder.mkdirs();

    ZipArchiveInputStream zais = new ZipArchiveInputStream(stream);

    Map<String, ExtensionEntry> extensions = new HashMap<>();

   for (ZipArchiveEntry entry = zais.getNextZipEntry(); entry != null; entry = zais.getNextZipEntry()) {
     if (!entry.isDirectory()) {
        String extension = FilenameUtils.getExtension(entry.getName());
        String basePath = entry.getName().substring(0, entry.getName().length() - extension.length());

        ExtensionEntry extensionEntry = extensions.get(basePath);
       if (extensionEntry == null) {
          extensionEntry = new ExtensionEntry();
          extensions.put(basePath, extensionEntry);
        } else if (extensionEntry.skip) {
         continue;
        }

       if (extension != null && extension.equals("xed")) {
         try {
            DefaultLocalExtension extensionDescriptor =
              extensionSerializer.loadLocalExtensionDescriptor(null, new CloseShieldInputStream(zais));

           if (selected != null && !selected.contains(extensionDescriptor.getId())) {
             // Skip already
              println "* {{info}}Skipping extension $extensionDescriptor as requested{{/info}}";
              extensionEntry.skip = true;
            } else if (installedRepository.getInstalledExtension(extensionDescriptor.getId()) != null) {
             // Skip already installed extensions
              println "* {{info}}Skipping extension $extensionDescriptor because it's already installed{{/info}}";
              extensionEntry.skip = true;
            } else if (extensionEntry.file != null || StringUtils.isEmpty(extensionDescriptor.getType())) {
             // Store the extension
              store(extensionDescriptor, extensionEntry.file, basePath, extensions, localRepository);
            } else {
             // Remember the descriptor for when we hit the file
              extensionEntry.descriptor = extensionDescriptor;
            }
          } catch (Exception e) {
            println "* {{error}}Failed to read extension descriptor $entry.name:";
            println ExceptionUtils.getStackTrace(e);
            println "{{/error}}";
          }
        } else {
          File extensionFile = File.createTempFile("extension", "", tempFolder);
          FileUtils.copyInputStreamToFile(new CloseShieldInputStream(zais), extensionFile);

         if (extensionEntry.descriptor != null) {
           // Store the extension
            store(extensionEntry.descriptor, extensionFile, basePath, extensions, localRepository);
          } else {
           // Remember the file for when we hit the descriptor
            extensionEntry.file = extensionFile;
          }
        }
      }
    }

   // Clean any remaining file without any associated descriptor
   for (ExtensionEntry entry : extensions.values()) {
     if (entry.file != null && entry.file.exists()) {
        entry.file.delete();
      }
    }
  }

  List<Extension> scanXIP(InputStream stream)
  {
    ExtensionSerializer extensionSerializer = services.component.getInstance(ExtensionSerializer.class);

    ZipArchiveInputStream zais = new ZipArchiveInputStream(stream);

    List<Extension> extensions = new ArrayList<>();

   for (ZipArchiveEntry entry = zais.getNextZipEntry(); entry != null; entry = zais.getNextZipEntry()) {
     if (!entry.isDirectory()) {
        String extension = FilenameUtils.getExtension(entry.getName());
        String basePath = entry.getName().substring(0, entry.getName().length() - extension.length());

       if (extension != null && extension.equals("xed")) {
         try {
            extensions.add(extensionSerializer.loadLocalExtensionDescriptor(null, new CloseShieldInputStream(zais)));
          } catch (Exception e) {
            println "* {{error}}Failed to read extension descriptor $entry.name:";
            println ExceptionUtils.getStackTrace(e);
            println "{{/error}}";
          }
        }
      }
    }

   return extensions;
  }

 def store(DefaultLocalExtension extensionDescriptor, File extensionFile, String basePath,
            Map<String, ExtensionEntry> extensions, LocalExtensionRepository localRepository)
  {
    extensionDescriptor.setFile(extensionFile);

    LocalExtension existingExtension = localRepository.getLocalExtension(extensionDescriptor.id);

   if (existingExtension != null) {
      localRepository.removeExtension(existingExtension);
    }

    localRepository.storeExtension(extensionDescriptor);

   if (existingExtension != null) {
      println "* {{warning}}Stored local extension $extensionDescriptor has been overwritten{{/warning}}";
    } else {
      println "* {{success}}Extension $extensionDescriptor has been added to the local repository{{/success}}";
    }

    extensions.remove(basePath);
    extensionFile.delete();
  }

 // Goes through the available files of the File Upload plugin, and returns the found xipFile.
  FileItem getXIPFileItemFromFormData() {
    XWikiContext xwikiContext = xcontext.getContext()
   // Get the file upload plugin, useful to get the input stream of the XIP being uploaded
    FileUploadPluginApi fileUploadPlugin = xwiki.fileupload;
   // Load the file list
    fileUploadPlugin.loadFileList();

   // We can use FileUploadPlugin#getFileItemData() which returns a byte[] and turn that into an input stream, but if we can have an input stream from the start, it might be better.
    FileItem foundFileItem = null;
    fileUploadPlugin.getFileItems().each { xipFileItem ->
     if (xipFileItem.getFieldName().equals("xipFile")) {
       if (xipFileItem != null && xipFileItem.getSize() > 0) {
          foundFileItem = xipFileItem;
        } else {
          println "{{error}}Failed to load XIP file : File object is null or empty{{/error}}";
        }
      }
    }

   return foundFileItem;
  }

 if (request.step && request.formToken && services.csrf.isTokenValid(request.formToken)) {
    InputStream xipFileInputStream;

   // Handle the case where we have a XIP that is uploaded
   if (request.step == "import") {

     if (request.filePath && request.filePath != "") {
       // Get the input stream of the file item form its path directly on the server
        File xipFile = new File(request.filePath);
        xipFileInputStream = new FileInputStream(xipFile);
      } else {
       // Load it from the form data
        FileItem foundFileItem = getXIPFileItemFromFormData();
       if (foundFileItem != null) {
          xipFileInputStream = foundFileItem.getInputStream();
        }
      }

      Set<ExtensionId> selectedExtensions = null;
     if (request.getParameter("allExtensions") != null) {
        selectedExtensions = new HashSet<>();
        request.getParameter("allExtensions").split(",").each { extension ->
         if (request.getParameter(extension) != null && request.getParameter(extension).equals("on")) {
            String[] splittedExtensionId = extension.split("/");
            selectedExtensions.add(services.extension.createExtensionId(splittedExtensionId[0], splittedExtensionId[1]))
          }
        }
      }

     if (xipFileInputStream != null) {
        importXIP(xipFileInputStream, selectedExtensions);
        xipFileInputStream.close();
      } else {
        println "{{error}}Could not find a XIP file to import{{/error}}";
      }
    } else if (request.step == "review") {
      FileItem foundFileItem = getXIPFileItemFromFormData();
     if (foundFileItem != null) {
        InputStream foundFileItemInputStream = foundFileItem.getInputStream();
       // When reviewing a XIP, we want to put it in memory, as it should not be re-uploaded afterwards.
        File xipFile = File.createTempFile("xip-importer", null);
        FileUtils.copyInputStreamToFile(foundFileItemInputStream, xipFile);
        foundFileItemInputStream.close();

       // Store the path to the file so that it can be re-used in the form
        xipFilePath = xipFile.getPath();

        xipFileInputStream = new FileInputStream(xipFile);
        availableExtensions = scanXIP(xipFileInputStream);
        xipFileInputStream.close();
      } else {
        println "{{error}}Could not find a XIP file to scan{{/error}}";
      }
    }
  }
{{/groovy}}

{{velocity}}
#if ("$!{request.step}" == "")
 ## Display the XIP upload form
  {{html clean="false"}}
 <form class="xform" method="post" action="" enctype="multipart/form-data">
   <dl>
     <dt>
       <label for="xipFile">XIP File</label>
     </dt>
     <dd>
       <input id="xipFile" name="xipFile" type="file" name="xipFile" required />
     </dd>
   </dl>
   <p>
     <span class="buttonwrapper">
       <input type="hidden" name="formToken" value="$escapetool.xml($services.csrf.token)"/>
       <button class="button" type="submit" name="step" value="review">Review XIP</button>
       <button class="button" type="submit" name="step" value="import">Import XIP</button>
     </span>
   </p>
 </form>
  {{/html}}
#elseif ($request.step == 'review')
 #set($allExtensions = '')
  {{html clean="false"}}
 <form class="xform" method="post" action="">
   <dl>
     <dt>
       <label for="xipFile">XIP File</label>
     </dt>
     <dd>
       <ul style="list-style-type: none; margin-left: 0; padding-left: 0;">
         #foreach ($extension in $availableExtensions)
           #set($extensionName = "$!{extension.id.id}/$!{extension.id.version.value}")
           #set($escapedExtensionName = $escapetool.xml($extensionName))
           <li><input type="checkbox" name="$escapedExtensionName" id="$escapedExtensionName"><label for="$escapedExtensionName">$escapedExtensionName</label></li>
           #if ($velocityCount > 1)
             #set($allExtensions = "${allExtensions},$escapedExtensionName")
           #else
             #set($allExtensions = $escapedExtensionName)
           #end
         #end
       </ul>
     </dd>
   </dl>
   <p>
     <span class="buttonwrapper">
       <input type="hidden" name="step" value="import"/>
       <input type="hidden" name="allExtensions" value="$escapetool.xml($allExtensions)"/>
       <input type="hidden" name="filePath" value="$escapetool.xml($xipFilePath)"/>
       <input type="hidden" name="formToken" value="$escapetool.xml($services.csrf.token)"/>
       <input class="button" type="submit" value="Import XIP"/>
     </span>
   </p>
  {{/html}}
#end
{{/velocity}}

Get Connected