Large XAR Import

Last modified by Anca Luca on 2021/03/18 11:28

cogAllows to import very large XARs which cannot be uploaded or imported from the standard Administration Import function
TypeSnippet
Category
Developed by

Ludovic Dubost, Thomas Mortagne

Rating
0 Votes
LicenseGNU Lesser General Public License 2.1

Description

This snippet allows to import a XAR saved on the File System of the server and import it page by page. This allows to import very large XARs which cannot be uploaded or imported from the standard Administration Import function.

Since 6.0 standard XWiki importer supports very big XARs which should make this extension useless in recent versions.

Install

Edit a page in wiki mode, make sure the syntax of the page is xwiki/2.1 and save it. The snippet requires programming rights to run.

For security, this page should be saved in an Admin protected space.

How to use

Once installed, indicate the full server file system path of the XAR you wish to import. A confirmation will be asked.

Snippet

{{groovy}}
import com.xpn.xwiki.*;
import com.xpn.xwiki.doc.*;
import com.xpn.xwiki.plugin.packaging.*;
import java.util.zip.*;
import com.xpn.xwiki.util.Util;

finalPageName = ""
exception = ""

def updateDoc(String newname, InputStream xml, String type, XWikiContext xcontext)
{
  try {
    def fdoc = new XWikiDocument()

    if ((type == "NOOVERVERSIONS") || (type == "OVERVERSIONS")) {
      fdoc.fromXML(xml, true);
    } else {
      fdoc.fromXML(xml, false);
    }

    int nbAttachments = fdoc.getAttachmentList().size();
    System.out.println("|____ Document: " + fdoc.getFullName() + "-" + fdoc.getLanguage() + "-" + fdoc.getDefaultLanguage() + " - " + fdoc.getVersion() + " - ${nbAttachments} attachments");

    if (newname == "")
      newname = fdoc.getFullName()

    def prevDoc =  xcontext.getWiki().getDocument(newname, xcontext);
    prevDoc.loadDocumentArchive();
    def prevArchive = prevDoc.getDocumentArchive();

    if (prevDoc.isNew() || ((type != "NOOVERVERSIONS") && (type != "NOOVERNOVERSIONS"))) {
      // We need to verify the document has the right name
      // Otherwise we might override something we don't expect
      if ((newname != "") && (fdoc.fullName != newname)) {
        fdoc = fdoc.copyDocument(newname, xcontext);
      }

      finalPageName =  fdoc.getFullName()

      if (type == "ADDVERSION") {
        if (prevArchive != null) {
          fdoc.setDocumentArchive(prevArchive);
        }
        // make sure that a version is created for this document upon save!
        fdoc.setMetaDataDirty(true);
        xcontext.getWiki().saveDocument(fdoc, xcontext);
        if (prevDoc.isNew()) {
          fdoc.resetArchive(xcontext);
        }
        if (fdoc.getDocumentArchive() != null) {
          xcontext.getWiki().getVersioningStore().saveXWikiDocArchive(fdoc.getDocumentArchive(xcontext), true, xcontext);
        }

        return 1;
      } else {
        def pack = new Package()
        if ((type == "NOOVERVERSIONS") || (type == "OVERVERSIONS"))
          pack.setWithVersions(true)
        else
          pack.setWithVersions(false)

        pack.setBackupPack(true)
        pack.add(fdoc, 0, xcontext)

        return pack.install(xcontext)
      }
    } else {
      return -1000;
    }
  } catch (Throwable e) {
    exception = e.getMessage()

    return -2000;
  }
}

def updateDoc(ZipFile zipfile, ZipEntry zipentry, String newname, String type, XWikiContext xcontext)
{
  def is = zipfile.getInputStream(zipentry)
  def ret = updateDoc("", is, type, xcontext);

  if (ret < 0) {
    if (ret == -1000) {
      System.err.println("* Document for entry ${zipentry} already exists and overwrite was not specified.")
      println "* Document for entry ${zipentry} already exists and overwrite was not specified.";
    } else if (ret == -2000) {
      System.err.println("* Document for entry ${zipentry} could not be imported with exception (${exception})")
      println "* Document for entry ${zipentry} could not be imported with exception (${exception})";
    } else {
      System.err.println("* Document for entry ${zipentry} could not be imported with error ${ret}.")
      println "* Document for entry ${zipentry} could not be imported with error ${ret}.";
    }
  } else {
    System.out.println("__ Document for entry ${zipentry} has been imported properly: [${finalPageName}].")
    println "* Document for entry ${zipentry} has been imported properly: [[${finalPageName}]]."
  }
}

if (request.get("filename") != null && request.get("confirm") == null) {

println """
{{html clean="false"}}
<script type="text/javascript">
function selectItems(classId, selected)
{
  var docs = document.getElementsByClassName(classId);
  var i;
  for (i = 0; i < docs.length; i++)
  {
    var doc = docs[i];
    doc.checked = selected;
  }
}
</script>

 <form action="" method="post" class="xform">
 <input type="hidden" name="filename" value="${request.filename}" />
 <input type="hidden" name="confirm" value="1" />
"""

  def counter = 0;
  def time1 = new Date()
  if (request.get("importall") == null) {
    def time2 = new Date()

println """
  <span id="selectDocsActions">
    <a href="javascript:void()" onclick="selectItems('selCheckedDoc', false); return false;" class="Exportlink">Unselect All</a>,
    <a href="javascript:void()" onclick="selectItems('selCheckedDoc', true); return false;" class="Exportlink">Select All</a>
  </span>
  <table border="0" cellspacing="0" cellpadding="0">
"""

    def file = new File(request.get("filename"));
    def zipfile = new ZipFile(file);
    for (zipentry in zipfile.entries()) {
      filename = zipentry.name
      if (!filename.endsWith("package.xml") && !zipentry.isDirectory()) {
        counter++;

println """
      <tr><td><input class="selCheckedDoc" type="checkbox" name="file_${counter}" value="${filename}" checked />${filename}</td></tr>
"""
      }
    }

println """
  </table>
"""

  def dur = time2.getTime() - time1.getTime()

  println "* Duration: $dur"
  } else {

println """
    Import all
    <input type="hidden" name="importall" value="1" />
"""

  }

println """
  <table border="0" cellspacing="0" cellpadding="0">
    <tr>
      <td>Type:</td>
      <td>
        <input type="radio" name="type" value="NOOVERVERSIONS" /> Do Not overwrite, use versions from the XAR <span class='xHint'>If document exists on the wiki will not be imported, otherwise it will be created with the history read from the package.</span>
        <input type="radio" name="type" value="NOOVERNOVERSIONS" /> Do Not overwrite, don't use versions from the XAR <span class='xHint'>If document exists on the wiki will not be imported, otherwise it will be created without history.</span>
        <input type="radio" name="type" value="ADDVERSION" /> Add version <span class='xHint'>If document exists on the wiki, a new version will be added.</span>
        <input type="radio" name="type" value="OVERVERSIONS" checked /> Overwrite, with versions from the XAR <span class='xHint'>If document exists on the wiki, it will be overwritten with history from the xar.</span>
        <input type="radio" name="type" value="OVERNOVERSIONS" /> Overwrite, without versions from the XAR <span class='xHint'>If document exists on the wiki, it will be overwritten no history.</span>
      </td>
    </tr>
    <tr>
      <td colspan="2" align="center">
        <input type="hidden" name="counter" value="${counter}" />
        <input type="submit" name="Preview" />
      </td>
    </tr>
  </table>
</form>
{{/html}}
"""

} else if (request.get("filename") != null && request.get("confirm") != null) {
  def file = new File(request.get("filename"));

  System.out.println("Start importing file [" + file + "]");

  def zipfile = new ZipFile(file);

  def type = request.get("type");

  def counter = 1;
  if (request.get("importall") == null) {
    def maxcounter = Integer.parseInt(request.counter)

    while (counter <= maxcounter) {
      def filename = request.get("file_${counter}")

      if (filename != null) {
        def zipentry = zipfile.getEntry(filename)
        if (zipentry != null) {
          System.out.println("Import ${counter}/${maxcounter}: " + filename);
          updateDoc(zipfile, zipentry, "", type, xcontext.context)
        }
      }

      counter++;
    }
  } else {
    def maxcounter = zipfile.size()

    for (zipentry in zipfile.entries()) {
      def filename = zipentry.name

      if (!filename.endsWith("package.xml") && !zipentry.isDirectory()) {
        System.out.println("Import ${counter}/${maxcounter}: " + filename);

        updateDoc(zipfile, zipentry, "", type, xcontext.context)
      }

      counter++;
    }
  }

  counter--;
  System.out.println("Finished importing ${counter} documents");
  println "* Finished importing ${counter} documents"
} else {

println """
{{html clean="false"}}
<form action="" method="post">
  <table border="0">
    <tr>
      <td>File to read from:</td><td><input type="text" name="filename" size="60" /></td>
    </tr>
    <tr>
      <td colspan="2" align="center">
        <input type="submit" name="Preview" />
        <input type="checkbox" name="importall" value="1" /> Import all
      </td>
    </tr>
  </table>
</form>
{{/html}}
"""

}
{{/groovy}}

Checking import progress

You can look at importing progress by tailing tomcat log file, e.g. executing command:

tail -f /var/log/tomcat7/catalina.out

Get Connected