XIP Importer
Last modified by Clément Aubin on 2021/03/18 11:28
Allows to import XIP packages into the local XWiki Extension repository. |
Type | |
Category | |
Developed by | |
Rating | |
License | GNU Lesser General Public License 2.1 |
Table of contents
Description
In a new page, copy and paste the following content. Then, access the page.
{{groovy}}
import org.apache.commons.fileupload.FileItem;
import com.xpn.xwiki.plugin.fileupload.FileUploadPluginApi;
import com.xpn.xwiki.XWikiContext;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.commons.lang3.StringUtils;
import org.xwiki.environment.Environment;
import org.xwiki.extension.Extension;
import org.xwiki.extension.ExtensionId;
import org.xwiki.extension.LocalExtension;
import org.xwiki.extension.repository.InstalledExtensionRepository;
import org.xwiki.extension.repository.LocalExtensionRepository;
import org.xwiki.extension.repository.LocalExtensionRepositoryException;
import org.xwiki.extension.repository.internal.ExtensionSerializer;
import org.xwiki.extension.repository.internal.local.DefaultLocalExtension;
class ExtensionEntry
{
DefaultLocalExtension descriptor;
File file;
boolean skip;
}
/**
* Import the extension located in a XIP package in the local extensions repository.
*
* @param stream the stream containing the XIP file
* @param selected the identifiers of the extensions to import or null if all extensions should be imported
*/
def importXIP(InputStream stream, Set<ExtensionId> selected)
{
ExtensionSerializer extensionSerializer = services.component.getInstance(ExtensionSerializer.class);
LocalExtensionRepository localRepository = services.component.getInstance(LocalExtensionRepository.class);
InstalledExtensionRepository installedRepository = services.component.getInstance(InstalledExtensionRepository.class);
Environment environment = services.component.getInstance(Environment.class);
File tempFolder = new File(environment.getTemporaryDirectory(), "xipimporter/");
tempFolder.mkdirs();
ZipArchiveInputStream zais = new ZipArchiveInputStream(stream);
Map<String, ExtensionEntry> extensions = new HashMap<>();
for (ZipArchiveEntry entry = zais.getNextZipEntry(); entry != null; entry = zais.getNextZipEntry()) {
if (!entry.isDirectory()) {
String extension = FilenameUtils.getExtension(entry.getName());
String basePath = entry.getName().substring(0, entry.getName().length() - extension.length());
ExtensionEntry extensionEntry = extensions.get(basePath);
if (extensionEntry == null) {
extensionEntry = new ExtensionEntry();
extensions.put(basePath, extensionEntry);
} else if (extensionEntry.skip) {
continue;
}
if (extension != null && extension.equals("xed")) {
try {
DefaultLocalExtension extensionDescriptor =
extensionSerializer.loadLocalExtensionDescriptor(null, new CloseShieldInputStream(zais));
if (selected != null && !selected.contains(extensionDescriptor.getId())) {
// Skip already
println "* {{info}}Skipping extension $extensionDescriptor as requested{{/info}}";
extensionEntry.skip = true;
} else if (installedRepository.getInstalledExtension(extensionDescriptor.getId()) != null) {
// Skip already installed extensions
println "* {{info}}Skipping extension $extensionDescriptor because it's already installed{{/info}}";
extensionEntry.skip = true;
} else if (extensionEntry.file != null || StringUtils.isEmpty(extensionDescriptor.getType())) {
// Store the extension
store(extensionDescriptor, extensionEntry.file, basePath, extensions, localRepository);
} else {
// Remember the descriptor for when we hit the file
extensionEntry.descriptor = extensionDescriptor;
}
} catch (Exception e) {
println "* {{error}}Failed to read extension descriptor $entry.name:";
println ExceptionUtils.getStackTrace(e);
println "{{/error}}";
}
} else {
File extensionFile = File.createTempFile("extension", "", tempFolder);
FileUtils.copyInputStreamToFile(new CloseShieldInputStream(zais), extensionFile);
if (extensionEntry.descriptor != null) {
// Store the extension
store(extensionEntry.descriptor, extensionFile, basePath, extensions, localRepository);
} else {
// Remember the file for when we hit the descriptor
extensionEntry.file = extensionFile;
}
}
}
}
// Clean any remaining file without any associated descriptor
for (ExtensionEntry entry : extensions.values()) {
if (entry.file != null && entry.file.exists()) {
entry.file.delete();
}
}
}
List<Extension> scanXIP(InputStream stream)
{
ExtensionSerializer extensionSerializer = services.component.getInstance(ExtensionSerializer.class);
ZipArchiveInputStream zais = new ZipArchiveInputStream(stream);
List<Extension> extensions = new ArrayList<>();
for (ZipArchiveEntry entry = zais.getNextZipEntry(); entry != null; entry = zais.getNextZipEntry()) {
if (!entry.isDirectory()) {
String extension = FilenameUtils.getExtension(entry.getName());
String basePath = entry.getName().substring(0, entry.getName().length() - extension.length());
if (extension != null && extension.equals("xed")) {
try {
extensions.add(extensionSerializer.loadLocalExtensionDescriptor(null, new CloseShieldInputStream(zais)));
} catch (Exception e) {
println "* {{error}}Failed to read extension descriptor $entry.name:";
println ExceptionUtils.getStackTrace(e);
println "{{/error}}";
}
}
}
}
return extensions;
}
def store(DefaultLocalExtension extensionDescriptor, File extensionFile, String basePath,
Map<String, ExtensionEntry> extensions, LocalExtensionRepository localRepository)
{
extensionDescriptor.setFile(extensionFile);
LocalExtension existingExtension = localRepository.getLocalExtension(extensionDescriptor.id);
if (existingExtension != null) {
localRepository.removeExtension(existingExtension);
}
localRepository.storeExtension(extensionDescriptor);
if (existingExtension != null) {
println "* {{warning}}Stored local extension $extensionDescriptor has been overwritten{{/warning}}";
} else {
println "* {{success}}Extension $extensionDescriptor has been added to the local repository{{/success}}";
}
extensions.remove(basePath);
extensionFile.delete();
}
// Goes through the available files of the File Upload plugin, and returns the found xipFile.
FileItem getXIPFileItemFromFormData() {
XWikiContext xwikiContext = xcontext.getContext()
// Get the file upload plugin, useful to get the input stream of the XIP being uploaded
FileUploadPluginApi fileUploadPlugin = xwiki.fileupload;
// Load the file list
fileUploadPlugin.loadFileList();
// We can use FileUploadPlugin#getFileItemData() which returns a byte[] and turn that into an input stream, but if we can have an input stream from the start, it might be better.
FileItem foundFileItem = null;
fileUploadPlugin.getFileItems().each { xipFileItem ->
if (xipFileItem.getFieldName().equals("xipFile")) {
if (xipFileItem != null && xipFileItem.getSize() > 0) {
foundFileItem = xipFileItem;
} else {
println "{{error}}Failed to load XIP file : File object is null or empty{{/error}}";
}
}
}
return foundFileItem;
}
if (request.step && request.formToken && services.csrf.isTokenValid(request.formToken)) {
InputStream xipFileInputStream;
// Handle the case where we have a XIP that is uploaded
if (request.step == "import") {
if (request.filePath && request.filePath != "") {
// Get the input stream of the file item form its path directly on the server
File xipFile = new File(request.filePath);
xipFileInputStream = new FileInputStream(xipFile);
} else {
// Load it from the form data
FileItem foundFileItem = getXIPFileItemFromFormData();
if (foundFileItem != null) {
xipFileInputStream = foundFileItem.getInputStream();
}
}
Set<ExtensionId> selectedExtensions = null;
if (request.getParameter("allExtensions") != null) {
selectedExtensions = new HashSet<>();
request.getParameter("allExtensions").split(",").each { extension ->
if (request.getParameter(extension) != null && request.getParameter(extension).equals("on")) {
String[] splittedExtensionId = extension.split("/");
selectedExtensions.add(services.extension.createExtensionId(splittedExtensionId[0], splittedExtensionId[1]))
}
}
}
if (xipFileInputStream != null) {
importXIP(xipFileInputStream, selectedExtensions);
xipFileInputStream.close();
} else {
println "{{error}}Could not find a XIP file to import{{/error}}";
}
} else if (request.step == "review") {
FileItem foundFileItem = getXIPFileItemFromFormData();
if (foundFileItem != null) {
InputStream foundFileItemInputStream = foundFileItem.getInputStream();
// When reviewing a XIP, we want to put it in memory, as it should not be re-uploaded afterwards.
File xipFile = File.createTempFile("xip-importer", null);
FileUtils.copyInputStreamToFile(foundFileItemInputStream, xipFile);
foundFileItemInputStream.close();
// Store the path to the file so that it can be re-used in the form
xipFilePath = xipFile.getPath();
xipFileInputStream = new FileInputStream(xipFile);
availableExtensions = scanXIP(xipFileInputStream);
xipFileInputStream.close();
} else {
println "{{error}}Could not find a XIP file to scan{{/error}}";
}
}
}
{{/groovy}}
{{velocity}}
#if ("$!{request.step}" == "")
## Display the XIP upload form
{{html clean="false"}}
<form class="xform" method="post" action="" enctype="multipart/form-data">
<dl>
<dt>
<label for="xipFile">XIP File</label>
</dt>
<dd>
<input id="xipFile" name="xipFile" type="file" name="xipFile" required />
</dd>
</dl>
<p>
<span class="buttonwrapper">
<input type="hidden" name="formToken" value="$escapetool.xml($services.csrf.token)"/>
<button class="button" type="submit" name="step" value="review">Review XIP</button>
<button class="button" type="submit" name="step" value="import">Import XIP</button>
</span>
</p>
</form>
{{/html}}
#elseif ($request.step == 'review')
#set($allExtensions = '')
{{html clean="false"}}
<form class="xform" method="post" action="">
<dl>
<dt>
<label for="xipFile">XIP File</label>
</dt>
<dd>
<ul style="list-style-type: none; margin-left: 0; padding-left: 0;">
#foreach ($extension in $availableExtensions)
#set($extensionName = "$!{extension.id.id}/$!{extension.id.version.value}")
#set($escapedExtensionName = $escapetool.xml($extensionName))
<li><input type="checkbox" name="$escapedExtensionName" id="$escapedExtensionName"><label for="$escapedExtensionName">$escapedExtensionName</label></li>
#if ($velocityCount > 1)
#set($allExtensions = "${allExtensions},$escapedExtensionName")
#else
#set($allExtensions = $escapedExtensionName)
#end
#end
</ul>
</dd>
</dl>
<p>
<span class="buttonwrapper">
<input type="hidden" name="step" value="import"/>
<input type="hidden" name="allExtensions" value="$escapetool.xml($allExtensions)"/>
<input type="hidden" name="filePath" value="$escapetool.xml($xipFilePath)"/>
<input type="hidden" name="formToken" value="$escapetool.xml($services.csrf.token)"/>
<input class="button" type="submit" value="Import XIP"/>
</span>
</p>
{{/html}}
#end
{{/velocity}}
import org.apache.commons.fileupload.FileItem;
import com.xpn.xwiki.plugin.fileupload.FileUploadPluginApi;
import com.xpn.xwiki.XWikiContext;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.commons.lang3.StringUtils;
import org.xwiki.environment.Environment;
import org.xwiki.extension.Extension;
import org.xwiki.extension.ExtensionId;
import org.xwiki.extension.LocalExtension;
import org.xwiki.extension.repository.InstalledExtensionRepository;
import org.xwiki.extension.repository.LocalExtensionRepository;
import org.xwiki.extension.repository.LocalExtensionRepositoryException;
import org.xwiki.extension.repository.internal.ExtensionSerializer;
import org.xwiki.extension.repository.internal.local.DefaultLocalExtension;
class ExtensionEntry
{
DefaultLocalExtension descriptor;
File file;
boolean skip;
}
/**
* Import the extension located in a XIP package in the local extensions repository.
*
* @param stream the stream containing the XIP file
* @param selected the identifiers of the extensions to import or null if all extensions should be imported
*/
def importXIP(InputStream stream, Set<ExtensionId> selected)
{
ExtensionSerializer extensionSerializer = services.component.getInstance(ExtensionSerializer.class);
LocalExtensionRepository localRepository = services.component.getInstance(LocalExtensionRepository.class);
InstalledExtensionRepository installedRepository = services.component.getInstance(InstalledExtensionRepository.class);
Environment environment = services.component.getInstance(Environment.class);
File tempFolder = new File(environment.getTemporaryDirectory(), "xipimporter/");
tempFolder.mkdirs();
ZipArchiveInputStream zais = new ZipArchiveInputStream(stream);
Map<String, ExtensionEntry> extensions = new HashMap<>();
for (ZipArchiveEntry entry = zais.getNextZipEntry(); entry != null; entry = zais.getNextZipEntry()) {
if (!entry.isDirectory()) {
String extension = FilenameUtils.getExtension(entry.getName());
String basePath = entry.getName().substring(0, entry.getName().length() - extension.length());
ExtensionEntry extensionEntry = extensions.get(basePath);
if (extensionEntry == null) {
extensionEntry = new ExtensionEntry();
extensions.put(basePath, extensionEntry);
} else if (extensionEntry.skip) {
continue;
}
if (extension != null && extension.equals("xed")) {
try {
DefaultLocalExtension extensionDescriptor =
extensionSerializer.loadLocalExtensionDescriptor(null, new CloseShieldInputStream(zais));
if (selected != null && !selected.contains(extensionDescriptor.getId())) {
// Skip already
println "* {{info}}Skipping extension $extensionDescriptor as requested{{/info}}";
extensionEntry.skip = true;
} else if (installedRepository.getInstalledExtension(extensionDescriptor.getId()) != null) {
// Skip already installed extensions
println "* {{info}}Skipping extension $extensionDescriptor because it's already installed{{/info}}";
extensionEntry.skip = true;
} else if (extensionEntry.file != null || StringUtils.isEmpty(extensionDescriptor.getType())) {
// Store the extension
store(extensionDescriptor, extensionEntry.file, basePath, extensions, localRepository);
} else {
// Remember the descriptor for when we hit the file
extensionEntry.descriptor = extensionDescriptor;
}
} catch (Exception e) {
println "* {{error}}Failed to read extension descriptor $entry.name:";
println ExceptionUtils.getStackTrace(e);
println "{{/error}}";
}
} else {
File extensionFile = File.createTempFile("extension", "", tempFolder);
FileUtils.copyInputStreamToFile(new CloseShieldInputStream(zais), extensionFile);
if (extensionEntry.descriptor != null) {
// Store the extension
store(extensionEntry.descriptor, extensionFile, basePath, extensions, localRepository);
} else {
// Remember the file for when we hit the descriptor
extensionEntry.file = extensionFile;
}
}
}
}
// Clean any remaining file without any associated descriptor
for (ExtensionEntry entry : extensions.values()) {
if (entry.file != null && entry.file.exists()) {
entry.file.delete();
}
}
}
List<Extension> scanXIP(InputStream stream)
{
ExtensionSerializer extensionSerializer = services.component.getInstance(ExtensionSerializer.class);
ZipArchiveInputStream zais = new ZipArchiveInputStream(stream);
List<Extension> extensions = new ArrayList<>();
for (ZipArchiveEntry entry = zais.getNextZipEntry(); entry != null; entry = zais.getNextZipEntry()) {
if (!entry.isDirectory()) {
String extension = FilenameUtils.getExtension(entry.getName());
String basePath = entry.getName().substring(0, entry.getName().length() - extension.length());
if (extension != null && extension.equals("xed")) {
try {
extensions.add(extensionSerializer.loadLocalExtensionDescriptor(null, new CloseShieldInputStream(zais)));
} catch (Exception e) {
println "* {{error}}Failed to read extension descriptor $entry.name:";
println ExceptionUtils.getStackTrace(e);
println "{{/error}}";
}
}
}
}
return extensions;
}
def store(DefaultLocalExtension extensionDescriptor, File extensionFile, String basePath,
Map<String, ExtensionEntry> extensions, LocalExtensionRepository localRepository)
{
extensionDescriptor.setFile(extensionFile);
LocalExtension existingExtension = localRepository.getLocalExtension(extensionDescriptor.id);
if (existingExtension != null) {
localRepository.removeExtension(existingExtension);
}
localRepository.storeExtension(extensionDescriptor);
if (existingExtension != null) {
println "* {{warning}}Stored local extension $extensionDescriptor has been overwritten{{/warning}}";
} else {
println "* {{success}}Extension $extensionDescriptor has been added to the local repository{{/success}}";
}
extensions.remove(basePath);
extensionFile.delete();
}
// Goes through the available files of the File Upload plugin, and returns the found xipFile.
FileItem getXIPFileItemFromFormData() {
XWikiContext xwikiContext = xcontext.getContext()
// Get the file upload plugin, useful to get the input stream of the XIP being uploaded
FileUploadPluginApi fileUploadPlugin = xwiki.fileupload;
// Load the file list
fileUploadPlugin.loadFileList();
// We can use FileUploadPlugin#getFileItemData() which returns a byte[] and turn that into an input stream, but if we can have an input stream from the start, it might be better.
FileItem foundFileItem = null;
fileUploadPlugin.getFileItems().each { xipFileItem ->
if (xipFileItem.getFieldName().equals("xipFile")) {
if (xipFileItem != null && xipFileItem.getSize() > 0) {
foundFileItem = xipFileItem;
} else {
println "{{error}}Failed to load XIP file : File object is null or empty{{/error}}";
}
}
}
return foundFileItem;
}
if (request.step && request.formToken && services.csrf.isTokenValid(request.formToken)) {
InputStream xipFileInputStream;
// Handle the case where we have a XIP that is uploaded
if (request.step == "import") {
if (request.filePath && request.filePath != "") {
// Get the input stream of the file item form its path directly on the server
File xipFile = new File(request.filePath);
xipFileInputStream = new FileInputStream(xipFile);
} else {
// Load it from the form data
FileItem foundFileItem = getXIPFileItemFromFormData();
if (foundFileItem != null) {
xipFileInputStream = foundFileItem.getInputStream();
}
}
Set<ExtensionId> selectedExtensions = null;
if (request.getParameter("allExtensions") != null) {
selectedExtensions = new HashSet<>();
request.getParameter("allExtensions").split(",").each { extension ->
if (request.getParameter(extension) != null && request.getParameter(extension).equals("on")) {
String[] splittedExtensionId = extension.split("/");
selectedExtensions.add(services.extension.createExtensionId(splittedExtensionId[0], splittedExtensionId[1]))
}
}
}
if (xipFileInputStream != null) {
importXIP(xipFileInputStream, selectedExtensions);
xipFileInputStream.close();
} else {
println "{{error}}Could not find a XIP file to import{{/error}}";
}
} else if (request.step == "review") {
FileItem foundFileItem = getXIPFileItemFromFormData();
if (foundFileItem != null) {
InputStream foundFileItemInputStream = foundFileItem.getInputStream();
// When reviewing a XIP, we want to put it in memory, as it should not be re-uploaded afterwards.
File xipFile = File.createTempFile("xip-importer", null);
FileUtils.copyInputStreamToFile(foundFileItemInputStream, xipFile);
foundFileItemInputStream.close();
// Store the path to the file so that it can be re-used in the form
xipFilePath = xipFile.getPath();
xipFileInputStream = new FileInputStream(xipFile);
availableExtensions = scanXIP(xipFileInputStream);
xipFileInputStream.close();
} else {
println "{{error}}Could not find a XIP file to scan{{/error}}";
}
}
}
{{/groovy}}
{{velocity}}
#if ("$!{request.step}" == "")
## Display the XIP upload form
{{html clean="false"}}
<form class="xform" method="post" action="" enctype="multipart/form-data">
<dl>
<dt>
<label for="xipFile">XIP File</label>
</dt>
<dd>
<input id="xipFile" name="xipFile" type="file" name="xipFile" required />
</dd>
</dl>
<p>
<span class="buttonwrapper">
<input type="hidden" name="formToken" value="$escapetool.xml($services.csrf.token)"/>
<button class="button" type="submit" name="step" value="review">Review XIP</button>
<button class="button" type="submit" name="step" value="import">Import XIP</button>
</span>
</p>
</form>
{{/html}}
#elseif ($request.step == 'review')
#set($allExtensions = '')
{{html clean="false"}}
<form class="xform" method="post" action="">
<dl>
<dt>
<label for="xipFile">XIP File</label>
</dt>
<dd>
<ul style="list-style-type: none; margin-left: 0; padding-left: 0;">
#foreach ($extension in $availableExtensions)
#set($extensionName = "$!{extension.id.id}/$!{extension.id.version.value}")
#set($escapedExtensionName = $escapetool.xml($extensionName))
<li><input type="checkbox" name="$escapedExtensionName" id="$escapedExtensionName"><label for="$escapedExtensionName">$escapedExtensionName</label></li>
#if ($velocityCount > 1)
#set($allExtensions = "${allExtensions},$escapedExtensionName")
#else
#set($allExtensions = $escapedExtensionName)
#end
#end
</ul>
</dd>
</dl>
<p>
<span class="buttonwrapper">
<input type="hidden" name="step" value="import"/>
<input type="hidden" name="allExtensions" value="$escapetool.xml($allExtensions)"/>
<input type="hidden" name="filePath" value="$escapetool.xml($xipFilePath)"/>
<input type="hidden" name="formToken" value="$escapetool.xml($services.csrf.token)"/>
<input class="button" type="submit" value="Import XIP"/>
</span>
</p>
{{/html}}
#end
{{/velocity}}