Changes for page Github wiki import
Last modified by Bart Kummel on 2021/03/18 11:29
Change comment:
Various improvements and fixes in script
Summary
-
Objects (1 modified, 0 added, 0 removed)
Details
- ExtensionCode.ExtensionClass[0]
-
- Description
-
... ... @@ -21,7 +21,7 @@ 21 21 * It would be better to use the [[Git API module>>https://extensions.xwiki.org/xwiki/bin/view/Extension/Git%20Module]]. However, this module only supports public Github repositories. 22 22 * The username and token should probably be input fields instead of being hard coded. 23 23 * Github wikis are less structured than XWiki, the following assumptions are made: 24 -** The ##_Sidebar.md## file is used as entry point. Only pages that are linked to from ##_Sidebar.md## will be imported 24 +** The ##_Sidebar.md## file is used as entry point. Only pages that are linked to from ##_Sidebar.md## will be imported. At the end of the script, files that are in the repository but not imported are listed. Suggested workflow is to add those files to the ##_Sidebar.md## at the Github side and re-import. 25 25 ** It is assumed ##_Sidebar.md## only contains list items. The hierarchy of the list items will be used as hierarchy in XWiki. 26 26 ** The hierarchical list in ##_Sidebar.md## can have items that are not links. In that case, an empty page in XWiki will be created to preserve the hierarchical structure. 27 27 ** It is assumed that ##Home.md## is the root page of the wiki. ... ... @@ -76,7 +76,9 @@ 76 76 import org.xwiki.environment.*; 77 77 import org.xwiki.rendering.syntax.SyntaxType; 78 78 79 -def CredentialsProvider myCredentials = new UsernamePasswordCredentialsProvider("<username>", "<token>") 79 +def CredentialsProvider getCredentialsProvider() { 80 + return new UsernamePasswordCredentialsProvider("<username>", "<token>") 81 +} 80 80 81 81 def Repository getRepository(String repositoryURI, String localDirectoryName) { 82 82 Repository repository; ... ... @@ -101,7 +101,7 @@ 101 101 if (!gitDirectory.exists()) { 102 102 // Step 2.1: Need to clone the remote repository since it doesn't exist 103 103 git.cloneRepository() 104 - .setCredentialsProvider( myCredentials)106 + .setCredentialsProvider(getCredentialsProvider()) 105 105 .setDirectory(localDirectory) 106 106 .setURI(repositoryURI) 107 107 .call(); ... ... @@ -118,6 +118,7 @@ 118 118 def sourceRepoName = xcontext.get("sourceRepoName"); 119 119 def rootPageName = xcontext.get("rootPageName"); 120 120 123 +importedFiles = []; 121 121 122 122 def fixInternalLinkAnchors(xwikiSyntax) { 123 123 // Fix to create internal links to anchors that acutally work ... ... @@ -138,6 +138,7 @@ 138 138 def fileFound = false 139 139 workTree.eachFileRecurse({ if (fileNameWithoutDirectory == it.name){ fileFound = it } }) 140 140 if (fileFound) { 144 + importedFiles += fileNameWithoutDirectory 141 141 document.addAttachment(fileNameWithoutDirectory, fileFound.bytes) 142 142 text = text.replaceAll(filePath, fileNameWithoutDirectory) 143 143 println "Uploaded file ${filePath}: ${fileFound.length()} bytes = ${fileFound.bytes.length} bytes} " ... ... @@ -189,6 +189,7 @@ 189 189 dir.eachFileRecurse({ if (fileName == it.name){ fileFound = it } }) 190 190 191 191 if (fileFound) { 196 + importedFiles += fileName.tokenize("/").last() 192 192 createOrUpdatePage(dir, rootPage, docName, fileFound.text) 193 193 } else if (emptyPage){ 194 194 createOrUpdatePage(dir, rootPage, docName, "") ... ... @@ -202,7 +202,7 @@ 202 202 println "== Source repo: ${sourceRepoName}: ${sourceRepoURL}\n" 203 203 def repo = getRepository(sourceRepoURL, sourceRepoName) 204 204 result = new Git(repo).pull() 205 - .setCredentialsProvider( myCredentials)210 + .setCredentialsProvider(getCredentialsProvider()) 206 206 .call() 207 207 208 208 if (result.isSuccessful()) { ... ... @@ -219,6 +219,9 @@ 219 219 if (sidebar.exists() && sidebar.canRead()) { 220 220 sidebar.eachLine { line -> 221 221 def level = line.indexOf('*') 227 + if (level < 0) { 228 + level = line.indexOf('-') 229 + } 222 222 if (level >= 0) { // skip empty lines 223 223 def links = (line =~ /\[[^\]]+\]\(([^)]+)\)/); 224 224 if (level > lastLevel) { ... ... @@ -240,6 +240,22 @@ 240 240 } 241 241 } 242 242 } 251 + 252 + println "== Ready importing ==" 253 + println "List of not imported files in the Github repo:" 254 + notImported = [] 255 + workTree.eachFileRecurse(groovy.io.FileType.FILES) { 256 + if (it.toString().contains(".git/")) { 257 + return 258 + } 259 + if (!importedFiles.contains(it.name)) { 260 + notImported += it.name 261 + } 262 + } 263 + notImported.sort().each { 264 + println "* ${it}" 265 + } 266 + 243 243 } else { 244 244 println "ERROR: Cannot open file ${sidebar}" 245 245 }