From fd4929b4d2ff4a4e68441ab96b5c8a7bce549ec6 Mon Sep 17 00:00:00 2001 From: Timothy Carambat Date: Mon, 21 Apr 2025 09:17:24 -0700 Subject: [PATCH] Feature/drupalwiki collector (#3693) * Implement DrupalWiki collector * Add attachment downloading and processing functionality (#3) * linting * Linting Add citation image small refactors add URL for citation identifier --------- Co-authored-by: em Co-authored-by: rexjohannes <53578137+rexjohannes@users.noreply.github.com> Co-authored-by: Eugen Mayer <136934+EugenMayer@users.noreply.github.com> --- collector/extensions/index.js | 26 ++ collector/extensions/resync/index.js | 53 ++- .../extensions/DrupalWiki/DrupalWiki/index.js | 320 ++++++++++++++++++ .../utils/extensions/DrupalWiki/index.js | 102 ++++++ collector/utils/http/index.js | 17 + .../DataConnectorOption/media/drupalwiki.jpg | Bin 0 -> 7293 bytes .../DataConnectorOption/media/index.js | 2 + .../Connectors/DrupalWiki/index.jsx | 190 +++++++++++ .../ManageWorkspace/DataConnectors/index.jsx | 7 + .../ChatHistory/Citation/index.jsx | 19 +- .../src/media/dataConnectors/drupalwiki.png | Bin 0 -> 24440 bytes frontend/src/models/dataConnector.js | 23 ++ server/endpoints/extensions/index.js | 21 ++ server/jobs/sync-watched-documents.js | 4 +- server/models/documentSyncQueue.js | 10 +- 15 files changed, 782 insertions(+), 12 deletions(-) create mode 100644 collector/utils/extensions/DrupalWiki/DrupalWiki/index.js create mode 100644 collector/utils/extensions/DrupalWiki/index.js create mode 100644 frontend/src/components/DataConnectorOption/media/drupalwiki.jpg create mode 100644 frontend/src/components/Modals/ManageWorkspace/DataConnectors/Connectors/DrupalWiki/index.jsx create mode 100644 frontend/src/media/dataConnectors/drupalwiki.png diff --git a/collector/extensions/index.js b/collector/extensions/index.js index 81a3a3dd7..087df6f21 100644 --- a/collector/extensions/index.js +++ b/collector/extensions/index.js @@ -154,6 +154,32 @@ function extensions(app) { return; } ); + + app.post( + "/ext/drupalwiki", + [verifyPayloadIntegrity, setDataSigner], + async function (request, response) { + try { + const { loadAndStoreSpaces } = require("../utils/extensions/DrupalWiki"); + const { success, reason, data } = await loadAndStoreSpaces( + reqBody(request), + response + ); + response.status(200).json({ success, reason, data }); + } catch (e) { + console.error(e); + response.status(400).json({ + success: false, + reason: e.message, + data: { + title: null, + author: null, + }, + }); + } + return; + } + ); } module.exports = extensions; diff --git a/collector/extensions/resync/index.js b/collector/extensions/resync/index.js index cb2595852..3ca1f44ab 100644 --- a/collector/extensions/resync/index.js +++ b/collector/extensions/resync/index.js @@ -2,7 +2,7 @@ const { getLinkText } = require("../../processLink"); /** * Fetches the content of a raw link. Returns the content as a text string of the link in question. - * @param {object} data - metadata from document (eg: link) + * @param {object} data - metadata from document (eg: link) * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response */ async function resyncLink({ link }, response) { @@ -24,7 +24,7 @@ async function resyncLink({ link }, response) { * Fetches the content of a YouTube link. Returns the content as a text string of the video in question. * We offer this as there may be some videos where a transcription could be manually edited after initial scraping * but in general - transcriptions often never change. - * @param {object} data - metadata from document (eg: link) + * @param {object} data - metadata from document (eg: link) * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response */ async function resyncYouTube({ link }, response) { @@ -44,9 +44,9 @@ async function resyncYouTube({ link }, response) { } /** - * Fetches the content of a specific confluence page via its chunkSource. + * Fetches the content of a specific confluence page via its chunkSource. * Returns the content as a text string of the page in question and only that page. - * @param {object} data - metadata from document (eg: chunkSource) + * @param {object} data - metadata from document (eg: chunkSource) * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response */ async function resyncConfluence({ chunkSource }, response) { @@ -76,9 +76,9 @@ async function resyncConfluence({ chunkSource }, response) { } /** - * Fetches the content of a specific confluence page via its chunkSource. + * Fetches the content of a specific confluence page via its chunkSource. * Returns the content as a text string of the page in question and only that page. - * @param {object} data - metadata from document (eg: chunkSource) + * @param {object} data - metadata from document (eg: chunkSource) * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response */ async function resyncGithub({ chunkSource }, response) { @@ -106,9 +106,48 @@ async function resyncGithub({ chunkSource }, response) { } } + +/** + * Fetches the content of a specific DrupalWiki page via its chunkSource. + * Returns the content as a text string of the page in question and only that page. + * @param {object} data - metadata from document (eg: chunkSource) + * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response + */ +async function resyncDrupalWiki({ chunkSource }, response) { + if (!chunkSource) throw new Error('Invalid source property provided'); + try { + // DrupalWiki data is `payload` encrypted. So we need to expand its + // encrypted payload back into query params so we can reFetch the page with same access token/params. + const source = response.locals.encryptionWorker.expandPayload(chunkSource); + const { loadPage } = require("../../utils/extensions/DrupalWiki"); + const { success, reason, content } = await loadPage({ + baseUrl: source.searchParams.get('baseUrl'), + pageId: source.searchParams.get('pageId'), + accessToken: source.searchParams.get('accessToken'), + }); + + if (!success) { + console.error(`Failed to sync DrupalWiki page content. ${reason}`); + response.status(200).json({ + success: false, + content: null, + }); + } else { + response.status(200).json({ success, content }); + } + } catch (e) { + console.error(e); + response.status(200).json({ + success: false, + content: null, + }); + } +} + module.exports = { link: resyncLink, youtube: resyncYouTube, confluence: resyncConfluence, github: resyncGithub, -} \ No newline at end of file + drupalwiki: resyncDrupalWiki, +} diff --git a/collector/utils/extensions/DrupalWiki/DrupalWiki/index.js b/collector/utils/extensions/DrupalWiki/DrupalWiki/index.js new file mode 100644 index 000000000..f42fe0cea --- /dev/null +++ b/collector/utils/extensions/DrupalWiki/DrupalWiki/index.js @@ -0,0 +1,320 @@ +/** + * Copyright 2024 + * + * Authors: + * - Eugen Mayer (KontextWork) + */ + +const { htmlToText } = require("html-to-text"); +const { tokenizeString } = require("../../../tokenizer"); +const { sanitizeFileName, writeToServerDocuments } = require("../../../files"); +const { default: slugify } = require("slugify"); +const path = require("path"); +const fs = require("fs"); +const { processSingleFile } = require("../../../../processSingleFile"); +const { + WATCH_DIRECTORY, + SUPPORTED_FILETYPE_CONVERTERS, +} = require("../../../constants"); + +class Page { + /** + * + * @param {number }id + * @param {string }title + * @param {string} created + * @param {string} type + * @param {string} processedBody + * @param {string} url + * @param {number} spaceId + */ + constructor({ id, title, created, type, processedBody, url, spaceId }) { + this.id = id; + this.title = title; + this.url = url; + this.created = created; + this.type = type; + this.processedBody = processedBody; + this.spaceId = spaceId; + } +} + +class DrupalWiki { + /** + * + * @param baseUrl + * @param spaceId + * @param accessToken + */ + constructor({ baseUrl, accessToken }) { + this.baseUrl = baseUrl; + this.accessToken = accessToken; + this.storagePath = this.#prepareStoragePath(baseUrl); + } + + /** + * Load all pages for the given space, fetching storing each page one by one + * to minimize the memory usage + * + * @param {number} spaceId + * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker + * @returns {Promise} + */ + async loadAndStoreAllPagesForSpace(spaceId, encryptionWorker) { + const pageIndex = await this.#getPageIndexForSpace(spaceId); + for (const pageId of pageIndex) { + try { + const page = await this.loadPage(pageId); + + // Pages with an empty body will lead to embedding issues / exceptions + if (page.processedBody.trim() !== "") { + this.#storePage(page, encryptionWorker); + await this.#downloadAndProcessAttachments(page.id); + } else { + console.log(`Skipping page (${page.id}) since it has no content`); + } + } catch (e) { + console.error( + `Could not process DrupalWiki page ${pageId} (skipping and continuing): ` + ); + console.error(e); + } + } + } + + /** + * @param {number} pageId + * @returns {Promise} + */ + async loadPage(pageId) { + return this.#fetchPage(pageId); + } + + /** + * Fetches the page ids for the configured space + * @param {number} spaceId + * @returns{Promise} array of pageIds + */ + async #getPageIndexForSpace(spaceId) { + // errors on fetching the pageIndex is fatal, no error handling + let hasNext = true; + let pageIds = []; + let pageNr = 0; + do { + let { isLast, pageIdsForPage } = await this.#getPagesForSpacePaginated( + spaceId, + pageNr + ); + hasNext = !isLast; + pageNr++; + if (pageIdsForPage.length) { + pageIds = pageIds.concat(pageIdsForPage); + } + } while (hasNext); + + return pageIds; + } + + /** + * + * @param {number} pageNr + * @param {number} spaceId + * @returns {Promise<{isLast,pageIds}>} + */ + async #getPagesForSpacePaginated(spaceId, pageNr) { + /* + * { + * content: Page[], + * last: boolean, + * pageable: { + * pageNumber: number + * } + * } + */ + const data = await this._doFetch( + `${this.baseUrl}/api/rest/scope/api/page?size=100&space=${spaceId}&page=${pageNr}` + ); + + const pageIds = data.content.map((page) => { + return Number(page.id); + }); + + return { + isLast: data.last, + pageIdsForPage: pageIds, + }; + } + + /** + * @param pageId + * @returns {Promise} + */ + async #fetchPage(pageId) { + const data = await this._doFetch( + `${this.baseUrl}/api/rest/scope/api/page/${pageId}` + ); + const url = `${this.baseUrl}/node/${data.id}`; + return new Page({ + id: data.id, + title: data.title, + created: data.lastModified, + type: data.type, + processedBody: this.#processPageBody({ + body: data.body, + title: data.title, + lastModified: data.lastModified, + url: url, + }), + url: url, + }); + } + + /** + * @param {Page} page + * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker + */ + #storePage(page, encryptionWorker) { + const { hostname } = new URL(this.baseUrl); + + // This UUID will ensure that re-importing the same page without any changes will not + // show up (deduplication). + const targetUUID = `${hostname}.${page.spaceId}.${page.id}.${page.created}`; + const wordCount = page.processedBody.split(" ").length; + const tokenCount = + page.processedBody.length > 0 + ? tokenizeString(page.processedBody).length + : 0; + const data = { + id: targetUUID, + url: `drupalwiki://${page.url}`, + title: page.title, + docAuthor: this.baseUrl, + description: page.title, + docSource: `${this.baseUrl} DrupalWiki`, + chunkSource: this.#generateChunkSource(page.id, encryptionWorker), + published: new Date().toLocaleString(), + wordCount: wordCount, + pageContent: page.processedBody, + token_count_estimate: tokenCount, + }; + + const fileName = sanitizeFileName(`${slugify(page.title)}-${data.id}`); + console.log( + `[DrupalWiki Loader]: Saving page '${page.title}' (${page.id}) to '${this.storagePath}/${fileName}'` + ); + writeToServerDocuments(data, fileName, this.storagePath); + } + + /** + * Generate the full chunkSource for a specific Confluence page so that we can resync it later. + * This data is encrypted into a single `payload` query param so we can replay credentials later + * since this was encrypted with the systems persistent password and salt. + * @param {number} pageId + * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker + * @returns {string} + */ + #generateChunkSource(pageId, encryptionWorker) { + const payload = { + baseUrl: this.baseUrl, + pageId: pageId, + accessToken: this.accessToken, + }; + return `drupalwiki://${this.baseUrl}?payload=${encryptionWorker.encrypt( + JSON.stringify(payload) + )}`; + } + + async _doFetch(url) { + const response = await fetch(url, { + headers: this.#getHeaders(), + }); + if (!response.ok) { + throw new Error(`Failed to fetch ${url}: ${response.status}`); + } + return response.json(); + } + + #getHeaders() { + return { + "Content-Type": "application/json", + Accept: "application/json", + Authorization: `Bearer ${this.accessToken}`, + }; + } + + #prepareStoragePath(baseUrl) { + const { hostname } = new URL(baseUrl); + const subFolder = slugify(`drupalwiki-${hostname}`).toLowerCase(); + + const outFolder = + process.env.NODE_ENV === "development" + ? path.resolve( + __dirname, + `../../../../server/storage/documents/${subFolder}` + ) + : path.resolve(process.env.STORAGE_DIR, `documents/${subFolder}`); + + if (!fs.existsSync(outFolder)) { + fs.mkdirSync(outFolder, { recursive: true }); + } + return outFolder; + } + + /** + * @param {string} body + * @param {string} url + * @param {string} title + * @param {string} lastModified + * @returns {string} + * @private + */ + #processPageBody({ body, url, title, lastModified }) { + // use the title as content if there is none + const textContent = body.trim() !== "" ? body : title; + + const plainTextContent = htmlToText(textContent, { + wordwrap: false, + preserveNewlines: true, + }); + // preserve structure + const plainBody = plainTextContent.replace(/\n{3,}/g, "\n\n"); + // add the link to the document + return `Link/URL: ${url}\n\n${plainBody}`; + } + + async #downloadAndProcessAttachments(pageId) { + try { + const data = await this._doFetch( + `${this.baseUrl}/api/rest/scope/api/attachment?pageId=${pageId}&size=2000` + ); + + const extensionsList = Object.keys(SUPPORTED_FILETYPE_CONVERTERS); + for (const attachment of data.content || data) { + const { fileName, id: attachId } = attachment; + const lowerName = fileName.toLowerCase(); + if (!extensionsList.some((ext) => lowerName.endsWith(ext))) { + continue; + } + + const downloadUrl = `${this.baseUrl}/api/rest/scope/api/attachment/${attachId}/download`; + const attachmentResponse = await fetch(downloadUrl, { + headers: this.#getHeaders(), + }); + if (!attachmentResponse.ok) { + console.log(`Skipping attachment: ${fileName} - Download failed`); + continue; + } + + const buffer = await attachmentResponse.arrayBuffer(); + const localFilePath = `${WATCH_DIRECTORY}/${fileName}`; + require("fs").writeFileSync(localFilePath, Buffer.from(buffer)); + + await processSingleFile(fileName); + } + } catch (err) { + console.error(`Fetching/processing attachments failed:`, err); + } + } +} + +module.exports = { DrupalWiki }; diff --git a/collector/utils/extensions/DrupalWiki/index.js b/collector/utils/extensions/DrupalWiki/index.js new file mode 100644 index 000000000..81be6f362 --- /dev/null +++ b/collector/utils/extensions/DrupalWiki/index.js @@ -0,0 +1,102 @@ +/** + * Copyright 2024 + * + * Authors: + * - Eugen Mayer (KontextWork) + */ + +const { DrupalWiki } = require("./DrupalWiki"); +const { validBaseUrl } = require("../../../utils/http"); + +async function loadAndStoreSpaces( + { baseUrl = null, spaceIds = null, accessToken = null }, + response +) { + if (!baseUrl) { + return { + success: false, + reason: + "Please provide your baseUrl like https://mywiki.drupal-wiki.net.", + }; + } else if (!validBaseUrl(baseUrl)) { + return { + success: false, + reason: "Provided base URL is not a valid URL.", + }; + } + + if (!spaceIds) { + return { + success: false, + reason: + "Please provide a list of spaceIds like 21,56,67 you want to extract", + }; + } + + if (!accessToken) { + return { + success: false, + reason: "Please provide a REST API-Token.", + }; + } + + console.log(`-- Working Drupal Wiki ${baseUrl} for spaceIds: ${spaceIds} --`); + const drupalWiki = new DrupalWiki({ baseUrl, accessToken }); + + const encryptionWorker = response.locals.encryptionWorker; + const spaceIdsArr = spaceIds.split(",").map((idStr) => { + return Number(idStr.trim()); + }); + + for (const spaceId of spaceIdsArr) { + try { + await drupalWiki.loadAndStoreAllPagesForSpace(spaceId, encryptionWorker); + console.log(`--- Finished space ${spaceId} ---`); + } catch (e) { + console.error(e); + return { + success: false, + reason: e.message, + data: {}, + }; + } + } + console.log(`-- Finished all spaces--`); + + return { + success: true, + reason: null, + data: { + spaceIds, + destination: drupalWiki.storagePath, + }, + }; +} + +/** + * Gets the page content from a specific Confluence page, not all pages in a workspace. + * @returns + */ +async function loadPage({ baseUrl, pageId, accessToken }) { + console.log(`-- Working Drupal Wiki Page ${pageId} of ${baseUrl} --`); + const drupalWiki = new DrupalWiki({ baseUrl, accessToken }); + try { + const page = await drupalWiki.loadPage(pageId); + return { + success: true, + reason: null, + content: page.processedBody, + }; + } catch (e) { + return { + success: false, + reason: `Failed (re)-fetching DrupalWiki page ${pageId} form ${baseUrl}}`, + content: null, + }; + } +} + +module.exports = { + loadAndStoreSpaces, + loadPage, +}; diff --git a/collector/utils/http/index.js b/collector/utils/http/index.js index 2283c0695..3648ec58e 100644 --- a/collector/utils/http/index.js +++ b/collector/utils/http/index.js @@ -12,7 +12,24 @@ function queryParams(request) { return request.query; } +/** + * Validates if the provided baseUrl is a valid URL at all. + * - Does not validate if the URL is reachable or accessible. + * - Does not do any further validation of the URL like `validURL` in `utils/url/index.js` + * @param {string} baseUrl + * @returns {boolean} + */ +function validBaseUrl(baseUrl) { + try { + new URL(baseUrl); + return true; + } catch (e) { + return false; + } +} + module.exports = { reqBody, queryParams, + validBaseUrl, }; diff --git a/frontend/src/components/DataConnectorOption/media/drupalwiki.jpg b/frontend/src/components/DataConnectorOption/media/drupalwiki.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3bf9eeb032c11b3aa64a5c0b3b4616bff781fe16 GIT binary patch literal 7293 zcmeHLX;f3^y55S2GKvsU5mH4I8Ol)^>O^c6&{9uC2oVrcMMOyXqPpbQ3;Nr&WGOh`a_6MJ`@lg* zC+9^H=-Bv#aBhBKQ6!c~ zWlJEHznTO7{?(xWWREVeXX&zKD$CS9+OuS77+5O0%TzaRU%vWtSGAKNdYg89w?hAm zga-w$R+{WQCcvFK->kl7v(2EX@S{n;TJ*0OboGB^(QgL*W)Ic_X{sy%0aMY12$0e( z>y{bx|BL#WqvdkUkQrboEi)*0{=3ji3rST3R~_gSY~dF z!=P711O*AY+=u&+Dd^y-V$j&5a54rhe}_Rrcev7PvC)znF>;5>Lrrp>mB#}>)z5u z0YB7O-Sab>zkq*aLmn0@Eb%z_UBln*#0#F$Nf>m%t^k9+Yk`HNDVT*Lj1CUL`S&nr zv|Dz~kwJl9zY>y_nmskB+F`JZD+Xz(y`e1Zq~aw;Tfw=VWd&|1LrvCAM58kQg-zUl z-DBW?cMJb>FivqU+t)di%8w_Vrbs6*wpc6Ycvexl^4*F+T3p747cPr;_JtAsU2<** zdEzRjZdugVoaQ{UQF85Vh-9m#XVM!)zq&e;u18t7q(kFh+%o%d>Bp5fffo*iphKiQ z)+4wkzD6FuA78J-{2qf|hw`oED^9>;)lal)A2FjmE;a5P_RL2I=iL zCcb3-B;ZytagsGs+TLA!AMXP%eLapi%pE#}K^0!b^aBDNKJj{fPn6%!4koQuZHkun z;}^`?3XPy|H~Zy7QSAaes@^T648CbE&y7EVFpF$^*rJ@{G*`Ggl_6HF^g=uc0hFH1 zpYcV!XqdCth_y2KYm$B1k=pLCv%KkCG2C+^AGS~GR+vlj`Lt`sw$Fs*>zrK|^Uu_Z zH{a|cQ~2?nk|d<$tcol&t4vMgpvUbd(Dt@(7c1IMhu0`}@T!_*21Hc#W1*6N2#=|h#oP!@Z?JIV%IzKm_Md2;>Ej>+ zwz3Blb+ilJfYwS_#y`eA-*>EW*E*z!l7uYa*i`dIs$YR|bZPqp_M)^;I zZ0fnX#qFf`JabM|oZK#)it1)w%fMfMTKhDcT1)6p;KE~jsLi2LT&(0rI%(xDxSY9o6>J<4)n2}ViDqkbshPThHW$qSpT!lJF*cgT}4HrIN zDLdp-7aFl42%VWt#~`CVN^VG2o_XFhNzR$~tfsnI-}7^x#*^+2>(86DJ@z^#CN_C9 zyP5ob`DKYNKU#iU{iMDjROc-VgO)zUAkl_17<7HI%B6G|ZJ;REau#%^4f(Ld(P35i z)j|Ae@hasVg7_yXykOKmKRN{}z-06$GPNF&ZKpQLhQGN)_R2Y%5!3dqlo1q@?R3nc zIqR0i0JHh(KdaI@3}PZ5B7}J5^S9or(+~zZ*?#>_Z6;T-dJp>5 ztP2J`r-#Hcb~i~+yA!T z<7IZ$v-cKoL_5QiFhE*}*I%OGFaT8;TNhBR!)ud+1=0k`TMIP{H(Q@ zC{B@3t1V#QDz z=Xl&X_{|P(zuSIesT{{>=|l>*)>z@I9KN|$*;${wyYD%-&Xi?u&6PJRhcPHjnTJMS zRDQuD6>!TqNn1HD;{BtKHx4Hp5BBgvJuSqg=t@|4l@^D8o^efUsyc{RdA>I5{m!V7 zq#j9vqSyhEdkon0o^?sdeEIA{>q zi}W+b+_}s=+j-~d^1-HJ3E@CT0)uK;9qoic56@d;kUopzcrNYw?u`4?NR8mDBerRF zA4?)M1dmtj!l0Fe*Z9?*Su;sNUVMT$TRDo>QFf)Y%PMyKfRsf!1-MKeUw>~%kvnQc zd3W)p?eqm=>(@T5z9*c65y2KvBpo15+VJZG!gS{|%!_swrhxTvQdl_RrS#S#Pb~Pm zNr}~hqKbl>xz5s`q81;X6P_yx&HusV8$00>b3!+%mmIju=+y1PsP-q!mEm_=tJiB~ z6rS$Q4^B#jXY&!c6=QqdHs|k^ytM5pg$@SFyuIMmE5y}7UXDJ`CNzxZfLBte@X*`# z52VGI>6*c^%E9tbgZnMVf8j~Mk5lb)Q4DSpNz}FbJYj9)*(i$9F@i{pE{kMNMGo?& zGvngubtU^g?QmgA^cd6d!WMSvse*VO(Q<8%ghXa8st;-?ZBYYIW~H|y%DYC%{hd}p|3z$ zXD7cP_C&tH3+~ZRcNE)AMiGK;!=k2Hc&NU;@Ao%WB)dDX?}GDRlpgPtRYx@_XO)}f zfMMB<#MamUb_sL;p;+nO7_{sT1|{xk0Zy}u!k`CMQ8~ZrEG_@{+bNA<)2{~BR;*{V zBdcxpoK>($?dxaO3dMe@9spyTla%W^2iOW61t3UtvoM%fF4|+nn~loY4M~?HZWuIh zlVP4Xjo066S5Hs!-uHzq22sTcT##F@?7Hv zo}7ZdK~2#%2{kllZ~9NIpLUsJQ0p!Sno=Y5b2(LVZyeEC=rb}LZDTut*FFq+I;+$O ziinva@hMZt&CAJmr*AG=?0btWwAl4w(2s@WxC3epiEWQXFJB@8{DgCPY|Z&m_i<&0 zFGZmtiv=|F`Cv1jc+IP$OWfdBJV%h9;n|~mN)QL?BEhTGq{Ajs=|Jk!YO~g`mRpum z=eIPy7O$4P#cNN~IJR?L(;)Koy}p^+WM#vb=j9H<4bQFWb&qeuWT<3?%U80xnaOH`vI6DZ7Zp=Sen)DN_4EdYq)rDpyjc%^|a_ zTac6dA#$JHJ6fF}5sS(tsd=AoO<#67rOS^tYgI zM4vb;bg9xrO5kT*EV!uRbo@tbV0d)S)F@V88@$NV0e;uo<6e66O$0x(mP1QN!xaUU z7<8wm6jj}iC;yQJQwm@fMND1P_yggU$0@x4Zc^Y%eB~oX(3{LEkoq*^9J#F|8`Xtx zduiT1Tx?iPT}PPCQXmphk8+i7l9cEdvE_agEkZ9St>5t@xt;S4WjqM?J`q<{Vc4>G z9BJ?Gt@#!anD(8ZyBtW)*$Jy_~j$i0KZpes*%fT&mRmgMbOF}r-Yuq;|ckc}G#dv%@ZHXfO# z>>?YOcU=yLhesSTqP1;8NCo(Yw{%B&KsZzG#`k}kVi?RkQb_Vgr|!~uGVLA!@7K^R zKo7X^!=e|Rf(mLPHk^k$?W6#UjRIr%xgb>rtt1J2EALCgFUk@DwmuE)1u1Dndg|ko zl$0`S#dvU|(kVmxvj}*bbc=<7`S;_eOcXT}vx3grWJH_J?%hFr55o0ynmVIEY+%Lt zidzy2-&9DeoYk3-b8Xvy2(D4)mx2wAwa98Wg50Ws9!183P8IJX#MJju(oVAKRLt8j zC~ZfqtYWLyhSxTMuw}Lhqrz6n-Rmmi>5W}hx33Kd?mT8KKW5jCK{eof#_Z>7r@a(C z5*`q&E~vRA@t9Txo@4VT4C?$0C1KDzYAYbmR@^k!Ko%5w6b1mBx*<=%2nd5R|9^#a8zv};>G38 z>xxKaVFeCtl%1XnPp=XcP6*jE3rFloFUk{;(qT3TM?`Vwz-TPt9P<`xeItL7cC&v= z1(#N?GLg8VNulbE8)+q)R>g0A+Oj9~Oi=ZeeUqXRC7bRl_YCh2wK{5KUD;up;?le+ z@#3Y4K2N0F1lA*8arMs75(IOEYE{X`c)?712#abkXK5$8RQvKnAT&cb({yz=ds;)t ztn6RWG8DvZW%-`S6Vd_^o4fW3wZ&da^XQE-`&`MM$`aq0oY%!Ym(qib^2;m~+q0%! zDLWaj2nGJksq6udl6FmU1);;$+r9L5uuG`Cj%Eml!m&s!=mZzzp$oiK02vPmw<@oJ z4#Jx}JVlx4RQ4#p_+tlaeF}p-keN=Nphhr(PY($Ynqg3H<>V}(dn+LBztA^6=Ol2C zrJ(vnoJr=Uv30GDiCMWLd?U9NJ9gYPj6R5hvd!IVl2A9>Awf z)1nR5pNKCx@~JSvS-CTDgrbNHjZ&fLGu{}RqK!h)vC&LevCuPU?!%gb4X(~&N_i~+uIK`9K?bEL_Q_4SRt zsTF4eELWgfg;`2Gl`=Tx?{)DMGlRHHV^ zb>Ls5VE(e4jo$H+R#g1u_bhig0+Gibc>7;fmS5J}8{SDaOB7{)Xbn#WvB&x+MP`z9 z5uvtKU?!&6W7)d%@aA~x{7(CRTfE{=;-6_UQ^p3M*u5b1f?abll1-4DoV~(Z`0`^f za-|5a)S)j2^9}MKsDmGMM(v8}dj|T#s_pCie;n(H=7!hYw2_ljF=#f)3ymv6x=jYE zuTPq0g4Rfb#U}1_xkFU0^eV?Y3!X9ue_3O+^~-{m0pA2q@68rc^LJ`|{LhHQ)V>;j zD`l0B8pt>pbg2T!V?xYPMQ-%VJIgqZcJ4gPRra-~I^tY!`^{9Pet9g#=IS3)2?(aI(Q1M= zZNX=cyO5-?4z^v}(cfU|^Q4&-e2&m}rWeTO&ba&T@q4xYOe>1)*4pM?zim#LI4(2G zLbef3shee8`De?Va=h|!*ZZCmKi=)<+ub?@3iL^-C(z%6X%6cFy|C4k%0vvh#Xg$F z&GqHr2kY*=e!4!M#&_DW7eUwM4ziURt6<(ZL6S!pOk;d}q(DuH{_PQtOk{yFzw+&%Qy6+B5^>un*!IW zwS5cEj}Ro9M9_pspOa8A=!+`6SVlz6nUXV^(a3@&a6bGK81S*$%(5rD*ixyrVueHY zH4Mt!4l>|W1gzLetfa`+e1bs;7)68Oxs3v(--A;+a^=TZ`95&O`@VsjYT<#;7dPM) zQYNaR2QtVs#j0+FZCZ3K>WYJ5GAU`v6-TU}0<94WK y``GsCXL)&MBww6IPS&&gUP(yor+$~+w>6_1B0xL(B>p$@{d-gTfACue>;4ZY(9y>L literal 0 HcmV?d00001 diff --git a/frontend/src/components/DataConnectorOption/media/index.js b/frontend/src/components/DataConnectorOption/media/index.js index d18803fa6..23d62b8c5 100644 --- a/frontend/src/components/DataConnectorOption/media/index.js +++ b/frontend/src/components/DataConnectorOption/media/index.js @@ -3,6 +3,7 @@ import GitLab from "./gitlab.svg"; import YouTube from "./youtube.svg"; import Link from "./link.svg"; import Confluence from "./confluence.jpeg"; +import DrupalWiki from "./drupalwiki.jpg"; const ConnectorImages = { github: GitHub, @@ -10,6 +11,7 @@ const ConnectorImages = { youtube: YouTube, websiteDepth: Link, confluence: Confluence, + drupalwiki: DrupalWiki, }; export default ConnectorImages; diff --git a/frontend/src/components/Modals/ManageWorkspace/DataConnectors/Connectors/DrupalWiki/index.jsx b/frontend/src/components/Modals/ManageWorkspace/DataConnectors/Connectors/DrupalWiki/index.jsx new file mode 100644 index 000000000..cf0df49b8 --- /dev/null +++ b/frontend/src/components/Modals/ManageWorkspace/DataConnectors/Connectors/DrupalWiki/index.jsx @@ -0,0 +1,190 @@ +/** + * Copyright 2024 + * + * Authors: + * - Eugen Mayer (KontextWork) + */ + +import { useState } from "react"; +import System from "@/models/system"; +import showToast from "@/utils/toast"; +import { Warning } from "@phosphor-icons/react"; +import { Tooltip } from "react-tooltip"; + +export default function DrupalWikiOptions() { + const [loading, setLoading] = useState(false); + + const handleSubmit = async (e) => { + e.preventDefault(); + const form = new FormData(e.target); + + try { + setLoading(true); + showToast( + "Fetching all pages for the given Drupal Wiki spaces - this may take a while.", + "info", + { + clear: true, + autoClose: false, + } + ); + const { data, error } = await System.dataConnectors.drupalwiki.collect({ + baseUrl: form.get("baseUrl"), + spaceIds: form.get("spaceIds"), + accessToken: form.get("accessToken"), + }); + + if (!!error) { + showToast(error, "error", { clear: true }); + setLoading(false); + return; + } + + showToast( + `Pages collected from Drupal Wiki spaces ${data.spaceIds}. Output folder is ${data.destination}.`, + "success", + { clear: true } + ); + e.target.reset(); + setLoading(false); + } catch (e) { + console.error(e); + showToast(e.message, "error", { clear: true }); + setLoading(false); + } + }; + + return ( +
+
+
+
+
+
+
+ +

+ This is the base URL of your  + + Drupal Wiki + + . +

+
+ +
+
+
+ +

+ Comma seperated Space IDs you want to extract. See the  + e.stopPropagation()} + > + manual + +   on how to retrieve the Space IDs. Be sure that your + 'API-Token User' has access to those spaces. +

+
+ +
+
+
+ +

+ Access token for authentication. +

+
+ +
+
+
+ +
+ + {loading && ( +

+ Once complete, all pages will be available for embedding into + workspaces. +

+ )} +
+
+
+
+ ); +} diff --git a/frontend/src/components/Modals/ManageWorkspace/DataConnectors/index.jsx b/frontend/src/components/Modals/ManageWorkspace/DataConnectors/index.jsx index 82560b433..4e6470b28 100644 --- a/frontend/src/components/Modals/ManageWorkspace/DataConnectors/index.jsx +++ b/frontend/src/components/Modals/ManageWorkspace/DataConnectors/index.jsx @@ -5,6 +5,7 @@ import GithubOptions from "./Connectors/Github"; import GitlabOptions from "./Connectors/Gitlab"; import YoutubeOptions from "./Connectors/Youtube"; import ConfluenceOptions from "./Connectors/Confluence"; +import DrupalWikiOptions from "./Connectors/DrupalWiki"; import { useState } from "react"; import ConnectorOption from "./ConnectorOption"; import WebsiteDepthOptions from "./Connectors/WebsiteDepth"; @@ -40,6 +41,12 @@ export const getDataConnectors = (t) => ({ description: t("connectors.confluence.description"), options: , }, + drupalwiki: { + name: "Drupal Wiki", + image: ConnectorImages.drupalwiki, + description: "Import Drupal Wiki spaces in a single click.", + options: , + }, }); export default function DataConnectors() { diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Citation/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Citation/index.jsx index b2a6f73f2..a4f3b4dd1 100644 --- a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Citation/index.jsx +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Citation/index.jsx @@ -15,6 +15,7 @@ import { YoutubeLogo, } from "@phosphor-icons/react"; import ConfluenceLogo from "@/media/dataConnectors/confluence.png"; +import DrupalWikiLogo from "@/media/dataConnectors/drupalwiki.png"; import { toPercentString } from "@/utils/numbers"; function combineLikeSources(sources) { @@ -197,14 +198,17 @@ function parseChunkSource({ title = "", chunks = [] }) { !chunks.length || (!chunks[0].chunkSource?.startsWith("link://") && !chunks[0].chunkSource?.startsWith("confluence://") && - !chunks[0].chunkSource?.startsWith("github://")) + !chunks[0].chunkSource?.startsWith("github://") && + !chunks[0].chunkSource?.startsWith("drupalwiki://")) ) return nullResponse; + try { const url = new URL( chunks[0].chunkSource.split("link://")[1] || chunks[0].chunkSource.split("confluence://")[1] || - chunks[0].chunkSource.split("github://")[1] + chunks[0].chunkSource.split("github://")[1] || + chunks[0].chunkSource.split("drupalwiki://")[1] ); let text = url.host + url.pathname; let icon = "link"; @@ -224,6 +228,11 @@ function parseChunkSource({ title = "", chunks = [] }) { icon = "confluence"; } + if (url.host.includes("drupal-wiki.net")) { + text = title; + icon = "drupalwiki"; + } + return { isUrl: true, href: url.toString(), @@ -239,10 +248,16 @@ const ConfluenceIcon = ({ ...props }) => ( ); +// Patch to render DrupalWiki icon as a element like we do with Phosphor +const DrupalWikiIcon = ({ ...props }) => ( + +); + const ICONS = { file: FileText, link: Link, youtube: YoutubeLogo, github: GithubLogo, confluence: ConfluenceIcon, + drupalwiki: DrupalWikiIcon, }; diff --git a/frontend/src/media/dataConnectors/drupalwiki.png b/frontend/src/media/dataConnectors/drupalwiki.png new file mode 100644 index 0000000000000000000000000000000000000000..2f8562f012cc56e689fb3f3a59a3bd22e98696c5 GIT binary patch literal 24440 zcmeEsQ+FMV^YuBgd1Bjanw&JYZCj0P+iYy7vC-IS?4+@63b5;Q;I)X1Ey&lKa9Q!GL>uNfrpG^C&OP00>G$U}TVBl!$b8&cXd;oyE$?G~ zp1aurUO0()=@qLpSA5JQT>Rv3N#Rdd2{XdW!^6YJ#;Z?8Y$Df%*>PHZSJ)q?$6wP6 zz`CM1bx9%shT{}29OIk-lcuk=6KN*c!1O4W{pBUUR8QIytt{DUk{p_3@so z;Z=RDqL$!D$of;CDL&qz^_WFu@4&7X#@0{(6jQCtI(18>wBKXjyot7C87&ew!u$H! z}r&oOB3+P^1sX60oub2%cW#uRyXEZ3SKeBi~=6&IDN|k(wez;A=IQ2pi*-fK&^AB7$$z=+IgLJpXGYyUw4iE z+(G~QXbR^#hRhClK{_$C9&k3RW7hiA78e$S@5Ec-^)_zygd$>b#n8zqHoG~s|4hE! zLG>Sc-!<@gn+sS0{zsYN+pxOTq^(xwvf0{Jf)xuoKv0B%uTK0~0>2U-G3lrhgdl|5 z0E88M!rF=T2;vcyp5B018l%NTvFs&0mKpg>F==w4(*e#+TXsV*q0gwj8va(VlyCEA z+?)hWTSF)=@l_|N5s}j8N;1H4So}LRK548cj?1Ou0%`)&^x2cp0W;EO2d26yW=a6q zY|2D%+^-c(Eibr{oX+=)JkXQhb&cHv>vaDA=&XvVtCXL(h=qGo=~gQJS_+( zwz*B>WpZLB634+*%?V|>)n-Fz_A!(KnN?LIXR9*-Moo=y$i@o*hp-;Hj5ubeCeq0S z+(UJ9Cx~z1RD29oZ@oXqVH{qbeVq#My-$2y3QU>vwQ?>1uj!Ljmkp)-c;vl%XuN~h zIrRuMHA{XKbHfn!X+pDX2gOZrZ-U_LRgkAXyK|rWsvalfq0vYw3U$W znLQEo7vanNf9&;|mk|i8)R_Q-MfizW*RGj=E~X7?1Q$h5@(F#$MrTh6p$otOHZgy{ zYcbv%3_STmn|zLKeX)F9kA1yty@wxm>Q5K_k2%#OYIgahqQ2Ksd9Jq~S~4L~M#H%2 z$}rn7EGCwPtBR90vS!||l~x@gc~dG;j0jw?fF5plh;E}(;Ga<>(H&VB zW=Ie_y|`V(I)93q{KmvzvP#o+xy(LsX7--|Aryd}x!vPXfQPS+B*v{)j9{P+s5eE9 z?k5(cerUOveK#pRYJG%>N8c8l7{P%-@jx$G!xAo1wP90tVM&yZ;x5kH6U#q(*AiNo zzZCL@33CggQiVmLR!wt>*yWC0rC*d?h(O$`t4iuJ=W&&!D>Hqgi~);S^liBx+*pSh zN;U!zXw{)uzkKNA+dzP?VB3;zD5!VY zJ}<$wh{nM{!D8j(wkk0!2x4Z+Dbp#On9&VHrXyWon#(CI?!=OwXa7z;MKpQjoQSeay-MvN2?Yadq4t^Y}Vm=1%HK` z5$`)F*A+P8Q-DbvYYN1w4gLC+p6LM&W|XEQ7TkQmq=m?YIvGV$IlT<&gb#lJ-1N~$?yEmsX} z+r+#vzmgP6%7jH0d>Ekld&+(QgQsqN)Wmm=V)D<`|ix@Qx;$QdSY`NlTv7 z*k!L^sZ2^O0lB+YH?DCZ!qu;XVALAwa7tfP;>6g9fgROIvzSI?8I@(5#r-D)*iRV@T>ux$*a9K!l?Zf>UBgI6$zhJ1L0)yUX&455 zU8|3*0gJ*lEK!@Xf6qyfk&Ik#psI}mQ zXY$FE)m&v#&|jcZ_k;kjEZ%c+5JnM@3z&%8c_Zr-2&2x0ID%@i$m@^vcG$(V{i|Ov z!KbqF!Si+?WhCoqZK6YCIrgAtb6ww58tvg(ax#$icmyXf#WQ*79nc5olZW}1S1~$ z7#CvKq5gEhbnm%g+>X%Har|9B3 zIvJwJ-C~mFnQy8?@_qj-uj+Ok;W|qm`hDp&D{c>PlYV?m3V+Acg6v%3N*zE-7hK1D z_CtzA-AO58!x%CcPV@pCUH#M4$>YUCo@g<-t%^W;21gc!j9}8V zgv_}P5T{}Fex_@BCxh6@w{;TlIUP>>0UETgxgDMXr zibf8wQXB4p;omU{>T%(45*#ut*$NIdT~gl8^Mzi^4{s`@z!AOj%b&ji8DZv&cCCRO zWKM*!-qRhgx*(~^@djq*iVNIJOr`n~W?7|+^z9KZEi_Vp@<}5TFL_-y!Q?X;eavb1 z;TDz^!?GyCu0j~{&0s5)PV)}uel>#6X4}ZBdWtP%qH$yuj&*>LSpyJkAzLVh2)u#T z8-L|IB|x_*vX?wcAc3~#xBksPSDntPrdob39_9Yeji?&KSYr-ih~4qFfn$b<5cy#e z7F9+!w&JKFiHIZ0?KAAr?BU-c@|)Wma@R_k7mB6NvR??Cl`z%_N7(xr@F~~EN4uUf ze{&J_6}ycj)R?tno&7!SYtC?vR_XXzjj7?J2CZ1cr~udrh%j#8Fs-(5ui#0xdC447 z7=xhE$@qQ}Jlh2N z$@j+1bA``g@p3?1PRN?YvAjKP>k$}b`R&=0gn0Nvb3AWA@C&Z{tA6R##N%SXx&fQY zt2p{s02*XqpbD-_3qv|3I=rQpLDmm=(Y-+;2Cw4X?{=#ggwR#k><2mGz$ZM zps>XUbY*Vj@EpP8bjRY-(EiTHodwA!Ih>-h*X8t6{F1w=E_NEnad4?swbr=9vd|rg zvr|BY3tJ#gmx<9tp$ zgnazsWpoi;GPOk;kGlEM7w>nStST-bAM)wGEZ=lHi^CLIC3W z%!5LhSI@lim;D}NbZjw+Wr>icFyIBbU^9HErrdA6i|B-aQniq4tK3J6R;375kA z{DV@1L5jf;R9;bC&If2lbM=$Zas9}cos0`1&Hk~}bB#xNtsM;eC)#j9d@B?5_)Gb> z&Nc2A49Y8=>;{WVF3I~p`wyCAfv;HsPrn+Z7^gl5TEn4=1ur7=DRl!e-}?e_0JULc z=Ah)FVs(y>R3LH?bukE!w>5bqsw@NDgu+VAJ$JG3NPg7k{pc_&iW1f;+74NUKlMWx`WQ!b8$9Q|wo>kVpCmG!>jY=&w@l;WlrxfCy=X zgsFvZ5mhT8zld>&0wTIorzppl`K$WkV74A44Pt+@E4p4K)Mn%hJ|%X~RCy#6y1~RRuT_ zKTtUC$lf2(1TBFf(`Ji_`5n?|fXNk1>o6(&U=3FkxUCFPVz%{IoPjL*=Q1~o z))gME7NlPF6BdR&ERMPQg%NtB0GZ=E`0^%$bBo=wc!}N1NNkFf4hE(hUY@|}IO9J# zPDVS;@@JTh*YdD{CY&V6&h~kn)|OY}A{=zCLC%=jVg-r1x&q3`M6LQ=bD865@Q4@# z2CDdy&pqYzqjPehS$v^`5fPd9T++8fK7gDETn_e3Eg?4`%-1<=xis~vjY@zIK;hF3 z->*_w-JXkcIAo;dV}IZG^}5Mr*ZDEl^Yz%n_4;4&&36K-j$iqxYL<6?R_{-S? zMu@OU|ab3yB7N`-EK(?X^l1-dj2pC%a6nkIF%ir7yS=1^hB z8+Ih_0=OSVT_;rP9a4-rM8r7hY6}`41S$6^^T{IrsYV&BMGY<6*+r}`bHWGc=T%agpg$7Oa7BbWunaKb z2&@xCK_>ii-V@qE5)0JC53+ylihqfNiq1c#8iwD*8O>0+S!#B@xxCN=?!je)fB3dG zoLY(T*K)zSz1bME`{v6wHc-S{2PkgPW)Tvo$1yb?ZE3_*W7% zPiaQ#mcrV*3AT2$q|QKHm9YWOTyR84Wp;n?{8uPe-lQ#ojb*>l)Kn2NaypxrR$d2_8KL z*Y`e3N^qER>#4)oUcrJa8ajUk+NU-TjKV?yE1*U@PNDFOy~N zAW$Tj&+-k%rxBh#92YD3dG=gjbl?h9Kkay|}o z*DVb0d*$1dVQg{@4kX}>Z#$QbZ)A@3spwgJC%9rd0e9-B!L@OKE*`LrTT<+_?T%ap z1GR4mDAWK=N;sHQs1V~Y<;$Qe*d*|Zl#I5nm%SA~Fd|tL(6U^hunY*`>HWCYmY&jT z!YA4f6W1b(qi)APZP1`aKasSIRZ6+u%>QdB^*E0opcFGf-R-Pw`7vKeUj9HBU)XAk zWvxfV8c(1e)uoKJQVKR0SZhD-f*#>yre8ItsNd`5BR84kDWO}A8y<-DOZvRaY1aSj zKfDy-y}G$MDB-sn4+9{}P$}7&@lLviU=}QtYJXs7?kAaAVdyLa102^VRDAXO8T;8% zY_7>Hd`OpaFh-#Xcz+5I`t|{!wrzrG+ku3vnRH&ca%>!|k`w?|Ml?7<`rYEu423Y< z3AZ(Rgs^G>C{eYO@Kkn7Bp5nHQN(HF$PHIMs_W_}%(xqyp(@%YK$`(d$DL?yz%zHE z^kIDhzFt;+j2sff0dJhsQdcVE zWm!fbc#kda;}RB~X!L%@(9Yz)4tgvmBp%Po;_5*RCx!&viP`%R<4KDt!*C12G_2II z&dwGmXla>M<*?&tdy24*DZQ_0pG7uv0ObMXmXYou?HL{eCFQ>}SFK`8$MDK(BaWVk z0Ex+TC5O^rqje7-1miPmfGJaK`DP`-l78McNhz#AaMb7X5lk;_`h`U_V{`%};^AYxcb-OB zCdJw+YL|Y1S&p!Hz~DB;Sa4t;_S|^IW%GB?BG<8?Wd(N9kmEXxk_(_Gu9*3@Xm-|Q zJ5JzhiD7HwYc@@w8)U&3nU8^66Emw_5murWz87?lj+7)M-*}9?Gag^KGxg)O*N|P@ z^Vha6)g<#PW6AQ!v&a3FEx@4#1|H8lCB9!Pq0PRx0tT)(-N!+@QEZ+d*vGjPBcZkF7_0OQDuQqn{1an z#W3+AqpL(`QSJNnLVxd+ z1)};WOM8Op2cdwLvoAzZrgwDEOHQ_3G;T(P_LP=|NBMSV6j}7`>Jh9(%?$w^wIcD* zpC^#UZi3TmVaueE+NYY$#BIP0=PuX{H2hII-M=sF7TE*j6@};UTN>kG*Sa(G5XK6| zE^&q48i?+{sR@r9l@JJ@it2$M1daP_Nw7n%Xy-TI7Hc%=K`z`3WSv-|URMnc0FJQ9JKVWG#~hGf8&JY-;7nU9_3Rqo zQqQL{;mkTI{r@e}P8*h_q$*hpUf#unLbTf#=qhEQgMyn@k@8lYK#r@1&z_ug#H(^*9f z8LZH8^?qZSp%>h33!!m=4p@Jf_;GUwRU8M2`;;6>?Wy9V4-J5+ri*ht)S2+jhtBoFv=$vml+gAg4dJ?3gt z)7MNdHtW)P10WMI^%=E?mE&Zsq{vmU0kpYDh-{NGnYf@KdioO&<^n1ZGWs7te-3bI zuAo?IsnJG{pF>w_Kl9yx_}qwp)qyy z-031Hvj}T$`DM!!Frcn&OFXXZ4^{U}tYEX}uhV3kAsTr)!dJyOpH{yXz_u22#X0Wc z0NMNQq;G1EZFYm*xj8wpYX*1`H2ljJQ(~GkrK4&K;Or=6lXJBj???C{9x^R3WN2r5 zVc&n}p`3pE&GPDw%dUs*{ru8G##t!zMRLtFe>eb&9yuyMUS5y{&k9WMH6Y*v1OWFHbHjqW6{d1bV(H1JXlZHjxKkW0?K94+ApM9^u50^4gmNyB zxAR&0Ds+^Sk}J^1ahy87Mc8M@ldXaPY5iUAN%e>Sc08W%B!4G6C7EQt-YzEBzw@rr zx6e?No)ZS(hP;l_GAM1^HJ{hVM#mCavI4&+Fy#+d`szmdEA^U1ld7+;YpnG~bX3*jjS#?okMOKJm0$ck? zHrXMrp@v`jW;M?TwjvP|RcuPG;g4x!>DQ;)w3WQkRXy?iJ@FPWajwFfq2%;&^1p6a zK56{<)r5n^jNe~nYNMA8ywr$u2{(Db+|T>N!_D!5fBYPHxj%<#^WMGmQ+hJf4hUwV z<-lblX5ZHmVd)^yYVedc%9{-!ncs9GqXa^mZ!MWlaEgo@Qq)rVt13FfN{|SAKFh<; z;|;$PAIWh4NL&Ii!u3OerA!HX=_^Y?uIyFz|CYTh5|lsG`%YSEhiiaI@H0vTPa+<9 zLFT-8#sS|{s!G3QrVk5kjCA;rJ*9bB`cmGw`rLZlVnhFeniT~vm(`R+dBg?IUtv7?X z_gCC5ZO5NggPw2dt2Dzget&2m)@Oi|2?l9KQ6Uqy+$@E|hKxx%w|Wjhd@uclB_BkX z#}Q2=dgWP10eKD=FbPFQYaQ8EhO{zwl(HW1W_+$Y7 zKZMIy{?Cw|N&3;S)KO}Q)F-s+v$%DQc@i%sGxd*6Nn0@T?`kPQl+tPq2mt-j1{r4b zBhEnBMqr-7gxsgC;`Sxv^Cdji0DcD)po=OXr|_O+Dj0G*z!EpT_xHw?)U4ZDkqExX zu;PZd3ps5uMceU$K?jqP=ccZLkEf7q@4rilL8ZO&f0TJ*keY;U%df_e6IW8mx89`Q z1*lEL}fR6Caav<-Vw+jPDU!S|X&+&_~0_r$cA(;sLy42#JOdUTV$(VsOC? zvqX+2jV#>(EuDyk-BiUVt9p%MsQgHH>#|fj%x@ReFEM{g{AV>AqJ{oBQhEV;lV|vL zb;;QH%CpSryYqHA7h&tVYG zq&2md3XVUnsnqqItKz!!!gsJ35^%S(3*65f%aE5k$IOSk!3Y# zgR(hEX}CL>AuERcjzxUw`^(FMrn(AJE;iO?HoC(((qvXMt#Z%nY0y9H$6in0%7l3o z9r&$GQ!S{OA^A!IA*G$-^X-0XA22w)3}enTDlO!tw6%%vJJqK$)N*5mE)5=3CCEG6^u7J1ATE7BhV{t`i#i80iq-8J6GUb~RBw=mBjH+<6 zL1PxoHy3LW1|a4iBM5!iSC+Nii^_b&bK*hF5XG?1wK~V5gs1EIt>BzI$h9)-8h+}V zx)$xiu}g(-jJd!0<7={L8Gk@tWDNVbe!ZHMwuSG&c=S+co)% zLw0F`-dULAcK(6CZ@hQM8&_KkUybJ-ihM!GJa?|0CSTTmep#ppB$Sw%>0E& zc0N$51I3UK#(R|e&HfJR=xuMS{ye{AgKvCCW<5Ob?|Irjl=gy>0n}7CA8wSW87O<_ zspk4MFW&EFBfjJOH5`N6>@fJVb#sqyliC;Y!=Ga2O>rDN?Bqo6s>JPsK2SfY-j?I; z8@OR+Ne9p`MGtd5!)g%B$7!(xCeu@Exj9*+$MOeAe30G4LCSfxAUtQv9dV!SEG=z+ z>a!2~bb)T?AHeOcS$0QnFFQd`(d#`7x%zW&yLEBVAevCDr#NcWEocqveSE~^yY_-R zpE8on zlop-%3RjF=$@Voz&9sigZW5$lQgo3NSr}DRHXGzjiSqbb zCS4JqqGtK7WhtSi{}Q`TB(1mV(x-I}*9$8V>hWWIOgApM`R^R=AJ%Hy5n#!>V?~&u zi5DWt*96u;v1E~?%_D{dR7b=^qf`<`I2QNbP40|=)UOp^0~=&vHnWdwL}(Zz3IJDy z{Z0SsOD-$%OG29g{41y3C7)@;pe7n2DFlqU{ z$~b9aE5Sc3J7Ez;$rmW^Z0t(s*QHlk`}0vED&#yay%kN zmN5xigW^CC63u9zqJr>*rP9YPrf#ql&6&2@GfFI&ixY-*c7Dl9=?)GD5(RrZUClM& zrq^k?v@_{}ND081E7KlI2OZvuJUi$p8o4`m69OiM?v*4VAi($a$)JsXJI^rF8XX5j z?8k=7JbmKL6zzS$(}tjWE4?SyAEw1}+N;2Hx+B4m{ZvU+P?#qzKKBG)s^==`hdhxe z?1ORU?<`=rNot*<4Am@&fRb;ueOIu6Zjl)cR=&te-@iQ)=BY_h`-K7t5EZ}}tzWh} zXIhee;beP)5mZvkLQQ|_xk-tv(P7{33tfq&*o*6&{hb3o#pHQ@o|9Q}KaH6=D!{kc6@3D#%llqT{uW6x?6h8(F(YxXR-ZXC5&G=_9mv>TJ9n{d5rA zA+`{ZcOn?c1DSk8wj3@)!CIHi8)hXl?f8qLegJxX8tY!oVwwWOBjsP(nc?N7A4!i! zY~M{ISMsP(cPXmgMXP^;G*^b9rB`iPX702cmUEtm5%=xBZ12 z@H2j1P2STDAQUB7;~;L|XG;i_87xU-xrnky?VZ4Q`f%=+K6ZAwa@e9)@|lrIVkuE} zZ_4n|+e49d)j}wAj$u^^LT7$ZSX4|5EGv?R^5^ z(ZG9e*B)D1h`afd=l)Rp0{6VZisYb0uen1jx!i?!qxS{YU}Sh`{(m0@8^EsmyoMX| zCCz77c(FsI!D(2DjOX*42=La$SynGJw7r4bQngHtdDzrFfkSl}CIx_?Nv>$}D_oiSPE-4HRHbdV0jEE9!{|D0RA~g16R=wsv z37Cdr&_E23NKWkP@@*s>>?+I8_X7@mg2LXYW8CdmIh9Zs#IIs*Z`_Q=UEEDHF7p_b z2h&Q^t9FShp#!u_=0L}dUOpVywY^p1dm;pV6FmF&HjjgbwP@%LszgrTCVbK4uTPCG zQuT$Yj%P>NSSAsDf8s8@qYZYj1jbfBAvxl4;^*{aK9w(sk=sGQT_HPf2c-s~Sz^l1 za)9caCzKrk>b^zztW5oP4w|zc0lV+SlAw%K-K>RGtxjMSF>)>SozMh=6GL;*l1non zBBeZ6E1EOTsp2Y-zpbn~1DB#z4<6x%`Ba^T?##vYJS@P@!pyCj6}o%m#iv_jb6rRD zp%tJ0jNdR9U)+Y57eA3W!c`U9v)qrqAiV`8paTV@vuR&OmDwA#(f1Z7$Z`1WPh|ZXXGeu`puBU%_yU@l+t+UA?`^k6%RIC(-K(B<5y+5pbz;w0XUu3EamyMc@Zl;1o)>6 zjd4JNala{@1ZAl=&!;!~+Q@B4^boj~-i-P1qH#frW}WsJ?S@_7nreYjJd8%-e2ZmT z4(jOf-|W7^(cjc!9q|KP!bn)|y?(948{)=vKsDD!Q8QzO-2R3@vr0jY);$`TY3K6* ziIBbLY9XqWBNk#|JrQmQ3AlZCKFneA<I>&P3ywaN z7hIS1l(}e(wHTd0-4(z68U66m9+IB@Phfmq4S2V#RHTuU%M7;!QJ+ z?dYXAQoQ(umQKV?Yir3X=x?ZQ=%w5-vwz1b^D$Ub*rt50kPWS}(1=LGU&>@8WB|4O z%5BBpJ)feMKmkB>Ot+G@Avy@VJ+G@>TWmr%cHI4*H#}Mv%xt86|%bjn4e$h(Yz==9T*1_jZ-zUvt+JtyNWf zO7RN$3-?zL^+TH`?bIJ3-B`u~5HW{wez}%5s<8R4rO~;`g*z4^z$FNY z`KrwIpxg5djR}p^#tM z_O&&)ShNv5Se{Ot%Ib4gc?TTb-_w7-nYP4rXDTiC1>K2z(*4FV7P@`A)6-EX=ivR( zC__}0kSfN@c;}AXNaLl1_2qd8)mrsckav563(h;_ps%{?;$keH zd(7T9yHw=Ytn|q7JTyiq`bgx}kJX4xr2!m19 zQ(axIENAhIFWSTG#5&PF)YE&yjA>sR9Y{QSx|ZK%u0^*B<#ZgP**u0b9CJn(&VEA; zJ!an3=a{rTx+>4oN4tJ^RJQJc0Xoe7jLyb4SmuaeBM1w%J`0s_=4)og$|8?${}& zmJ-gyT973F9daQ8OkfeI3{ZQ{eA2R_a?3lP5{S=pVC8JNS&l9WyM=_-!mp2*L3a=W ztwY&@8`GIXM)GThvP{JnemFtga`YBMRU}JmIAlECeIvb&+^rFY^`?UQp}vKktB*F&zw%0_i?&wvC8zNNo=g8P^sVKM@^4 z>sc&YO}ihSBVl%M3iEEHxfpFGXYj+JaaJtOH~o*9nv~m3QKsm+Y~DGXV=+Kl?Pq$! za-|mt+2eq`w1DV~13-hwj1+1>Bd(t61_C!Jb-QtWQ8=KtcK^Yyr*ZA331xZcpD!EW z-LIfr7Me;LVbU*x*3x^z@kq>>{3bM26BJI8XKJj8zz)Kj{zm{k@@|sw;N2<}a-Su} zD8zs&=A8Oi8QJ?=XGwyJ<=hV(9wVL7^||x_{YHBFP@(CRuM1Je{iyQ*1v;M#!aW+Xqw8)2?qhYlQPePq_OS;#?&@Po z3s^pL#}v}M*X^7-y)70>r1#;E3p&WXbk68PlX?VB{7?s-Z%~FEf{DA=H)ieeUr;xg z|48M5BZ1me0fayTeL z+dx@xtb6L434$9x9f$4E-SR!UGA=YJq7pI^(lb4+kCE&{o&rbMI;1h^OVY(VGH>Ao z8Brc~9|Hz2hdAL5;*=k(HS1<*B6qzbiN#2C);3^n7eLj!#@WnXQ-|Y6bHBwDXLWaL zv471eoBa$lGHt7@iFen`g(UZSpq!XA=zjVqMo@vo7Hdz)bH7H#y$}7P1NkHneA5bq zJIRg-PC%=I963?kk8x5*V;w>18JK+OZe-6Lf^bx(cpB0Roq>;6>x9=(hVmYf!T3P) zvGmObjn0DoE~_5?E=LNXtSBI8G9pF((_N;fMu`|ISi1;{4m>BMcF)GE@xx{fDMeoM ze{@!FQ$2}ZrAS*PFn+1bS>0`1CpsZ7{!jHP>2Iqq@ICglKBLOI6SiT?mA(JE8S(EN z733TW)_Uz}oswFULjy+_NLy(xv$MP*QXQ*PyNgaWd;~VSCiI}p#kv9eYa6=<)-bR z^{zzsMGiBxGk-t<-efaXX|GYA0o7rep{TwKaBxyti1b71i zvUI3Q@R>NtR(B(Rke1$cvPR_RB+;AO--AjRE+7Dvf3++kqCv!f55bSCF|@{UnYBdm ziP6aIVQ4DJIY`8N%(!o`y;k;rRVOeW1jS0wTFWC+3JTl#(x%(GmYh$N?kR@U5Uz2e7Cox+O-EPsYP=#35$g(>5WKv=5kp+9C#Q9W@P&arR`g$~C zo7mFNV|sOhd-xC%4V+5u2AC1b@;m>zF2~U&KP=jr1I;lVJv&QsVagQnf%|A0tw1Y%!|CD;;!@l6gk3~ zu*~^b$Xxt;7_(oYS2(Zum1!7-aacy>DRt5K@my1w(-EyViS?|aZaL^iQpJb%az=-0 zn_05t@HW`84&R*CZz&Mb?PpIinopVIG4Gnp8uVItpfRUXnR$XUx6~vv8cYo4PJV0Abw7# zlf-4Kfxw75xQM#}&%_te(V)MNYz3C*f@uTDGmwbK577j4C8SKs0XW|XX(t)y$;+~- z)wJ?x!>8+-vFy8{JQ3nW1PGnvoW#l-aS%7T=?e32aF&Wa5>j?w0o@qqHc<MED8&<$YCY<>&#ID6N zMa_pvdN-^Rh%eT0;%8T{oC>onYlV4s-}|U-Hn~S(k%v3SA(Y$Xa&kf;~gHLqg`e22;Ama%)0K$$(ny&yCo zguj}>5f$KJ^4GTC^V9bG$fVPYCx>c5TP}ec%>r5Q(akenj~)QF8aTQXJkg@7&j;vk zaF#jbyt1lnl__%HUcKi#F8!LK;8&L`q~%}vx5eP1{pUahz=0;!&Svejwh+->pD{)h6^W`(u@G?G*TW5n4I^38_@^1AYW2*Tp@;cQ z8;kIuw!hR1ofiidT25~~3uUJ{Xt=}mG9PDhbY8a*WRx%20a1c{X{dC}HOwLntZ4DQ zSq^TEpr#{UrJ_FYOwBwI1D>%vw=nFg-n*|QvyHzc<11ha>hzxt;`-M0d~p!OhE=EY z@!%24;Tab@bblAAFS46Al{)dkO^@}o(?qf&`umI+$9}NAvl$pn{EwoPMC{fF6P6i} z9v%UMH7PUbd)^%Fn54Fran^Y+jyL3o2TpZ=Wg3g}-b@_?-*<74*TlSTKYAHZ9;LJk z{8QlU&5z#4Mf|`&0e@#Qxf?mGa>MC(wbI4|w2LEHvuGd(V3e?yxXI{sD<9u}ulmX4 zkw(Ej-?|-=>ue;-p4#fqjvq%|G+Pc0D4V60{|nI~y*C)Qfd|8EX?avkGXT#$+zQ*< zufuD*A2j$-Ra@)yNzs2Jc#|9>K;doET@ zzSe=n1vBJJ?SL8q{Y10>Z-Yro*RenlQt|SYZt#Q%a5QTsyQbzO2%n6%_o1fBy{x{H zMZH7O1r6}wV~I*LN|a5V`Z95cdnPP_@U4@jeXoC>n^qC}@+QfI% zs~AeRKGwMQ!owAoNl1u%LIiE|nWAa`ib!N0)DO#wl^v8B%1p41zJQ-kt(piBbbO+-JF~$W@kdzQ z_7jE?f?OnOP$6tTf(L+EVyy4TTZZo5UhXN2#%=cIE^0%T)}k<(siR=3qkhm<^EE#~ zs&9PmS|;k3bP(G48R1j%xn7;@HCHj}Pr2TpRHfDi&Tu3YG8YoSXt0n+6@J3{d;&K4 zVF3Klvo*Zw*-iMobGcJI1C8`oAZ(kpekc{)gRx$vQ*IwKPvfsVvL1YFxM4=$;oWo$ePq8U0|X`zqVY3|3lCx`5Lw?L+r2Q%CjGT zJpv9cF6zPh$Ow0w_aU00FXCaS{HATql{#N{h<+ zpX)T?%LX&}p^|EOR-MHt^yf+gPgt(s3QzI58%s-_I=FZL*Unuv#ldv#0v-av-F=YY z1b26LcX!v|F2M=TgG+FCcZa~>7GTie?gM1LyuV?e?!!LoT3y}MwYu-?eDK4?ps-w6 z`q)j3#7)5?ap+CbthSJb(f?r+Qa9xT><90QFBC7vRr?!xrhMbmp zhDmmxv0nBwdz8%@E@4lL*AP#-OeH-#(b)Cx?{9CtF0GbQ4Z$HTbeLl03mXBVpwP_8#q>%wDL3hSMX z(+2ag3E%6CvcPbFzXHAe=%RZ`of$8&R&}f>2A*Q7_21&3aSWcYHj#v;>297mEoaF? z8xmnnmY=d(q~{HeakN~cd^kJ=s6X38-0@4n%BxsYwf}a`=Nd?EXA$=m13YZ4_p}rJ z(74+_rjcQ5O&*uHNJNb-&4m@@q_TNU@^Dz`D{b;cRv6Ng9X`_=4QDdT zqPTOgKC7GfL*0m+YI<_QX*Z_5V8@-O?|I_4AlMKr?APh@T*h4ZK5fVGx+bplM(dW0 zHe@*h3J=qng);1F_lrY`Ovis*epIEN9Xn3#wj^4)`D!E>KF!_tb{~S6arna0(79bq zec(c==1Rmz1*XI>NIhcKFO7GlQPML^JHJm_qp+N-5Mgd+jdf&@9BGMt8zXqd#Far& z!htap%D3~d-CA$Ws&4Uk72hiJk@}oH57S^4HvByykcg9@uhyMd#TKM$njf%16!*;KQW&(bdj!{IpR43Koc3W6{ z8w{8YOcrz}2gC+45y$(jT(q4BpWaW}vcN;<#Uwt0GcBew)9=?Xh)^_4Vb_rl^Br(& zfsYL|km%c&qd>0S#CksX?I#st8el@))X?^UY?R*UtH?@wU0k`2PJ+(N*MbBK`nVXv zz=Y`RFFMCANUPC&fLz#II*K?CnvtT2McGOSdaS~DO+21QGYh_(K!IDZS*I-6dfsk0 zC@}J4nI5?%N#3HTE-QL9t7aNsh)#63Y%yK?R)47j{2Jg}yuN91Zuao-WaaTP`|qz} zPZ|~UTk(u;<5T=@BA0Dk!O}8Y=kf&g-_W;M5OtY|!FfXvJemqRt!y0)UWJt}#vAF) z2GMorIlTp%=UjrAdx`>zNqF!W1XzJ>j^y9kNu10^vZ*G7fLhyCAx2+WgyYhYm^n?l z+1B^X`j72%N`og~4gm1qoBc>My-lEz*_1Qvd}djbl}FfgPECD+a+o}56B&XLatxgA zNxuHF@e&NwIU$=zFqyeq=t*;W5Crs_GELuUV+ybJ{w3?gn!C*6?ez7D6l`c6Cmmz_ z%NDzz?0T^y*46Sdbqr*Fj;VE4*=m98!}a=;VMqyuMR|1T(2YmAJn#=fsI2mrqhCXQ z8N+aOCn1^pMAUj|ffd=&hUZHZ_npMOmHF8OjlFu_Jl|qK@C3kw(4i9T+ESgxILQ>; zS&fHy(?3-4qt!5SDu{T(J;M9YUEoI;#Oa!|7i5gd>*8qqerG52KI_i&x=K$n@EOC5 zgwGTNH}?&pvMWeJQqeBdU5<2^*C)Uo64B;+)3SFm>gv3mljpm9F|iDXGKHj5E2nJmE6?eH%knjbj@2<3cTdlMmRD8 z0nnY2$~X@He+frEJ5ZHH)c{yNE6Y4p9`V-RvbrELulNo+zbE)!mt*PIa2|iOPF?B| zN?*~XZMBa{GePSFFVF*FFqA8CX&~<4uw}Mwc7*k6yGN{K0PXUbQ=A@_s9t1LIqpp4 z-y`u(3gWXkaA*BzY?VQJT_LR`fa@oe62Pe$QHWz3G%$=5l*AAINL!E?+OxCr?G#LY@N5fvqK&&idcn}x}yZfe598Yl990=11?0ryC$33^! zM{OJxO89yAh7-phdZ5D7=}zb%V+Qvr!7#zU3wCD2w1V+fBrw<`6)pZhG#0sb5135{ z4_;1BCK9R|)>LshejgsO+N0WQb(gVH>ZFcQ2-DP}Dp=sWx|qz59gwT2w(L*u-i{o= z!X$Bb8R=F#p81}I42cq?BhsU%O{a8MH#HOAz`%*q6fVALiP0-+z_#sOHY+oHk#sp8 z7t}*gMT<0j&Ri~G0SuOHV=+4qrU6*e!}(rT-8f=p(=KX8SHIi}(amb@@SXPJ#S9{c zzEsi2(}UvaXFQd%$C?dD+-`VU`(P>oDz*rNx7aVFBZ}^ubNfMs*z6>`p3h;NwulWA zZilAgpneR}8+4<9heO|M2@#T0H6MRt#5w}yrYTigD9QKoE)G;J4v~sR_F4h~J;<#m zl7DSy^cJ$@JTk$NM|D6waEtXf+jO1)>KhU`!Pw^Wsk$P(gjFZ3 zY4mf6E~Z{@vRo7Ta3`Obn943h?Zz34SomNI6qY}lZ`k$ z&d8jbqpgh7MsyZ+U*q56c--tYah8bHVh=Wrivp3Uc_#ZcLd%F7S34xH9o@SHn9o1kYq*H@Ph62q$alNIMh z$qa35L}ZI#D}ncB!WG}>Slu$js1o*Ro9ip4HzZSJqCyzSch5Sdx86K9%0~XpSO=P} zRAl1{I&qqH8m{|$sbpXxZEWF*-xFi#%fc^1$JFOAJx>`92054BN@MzC0q!4SIh%=W z)Fcm4Au;XnN;dqVvabwo{nxd05u%3YkD}Aw|Aky%J+nl;b)oMMR1=^5cb=d7;3DeN zVQOq?@rrC!2`y^#(9)ZL5f=CjqWBdm2EfiLL34Ds@T?+XDyO>+Ox>X`*HayFV}~y~ zw~ZHpSOZ`tj}Scdd{MRcb(TI!cY-7+uzsoa8jF$;2$WH@mpwHV8N4~ops3qD6~f08 z;2dWJApSNO4d*}DtO(BWB$MnHVY(WQc$Y8P7mb&CFrJqYmAIZ0-5kYoEIV0; z3m#z27(}PLC}*_~;l9=b$lXYroNEVC-I2sd3%^Zar5a&&p}35{4=jcaY_96_BoZ9+ zPK4)qUw?JpDycLeg1}*k9RfSPN?Ff<2;X8v1}WcZpoIs)+@VA8EgUs?U_8B@Ka)9uO#Er2HWIEy8{F~q(9N|dEUIfQe({hg2csZGY80)|n zuRt?e>tMgF2KQV2I89BVocirU{vy$p8({!KD;()B6S1rCMY8|E1Vp?(VTM?uENc$u^*ytb{xq0fljPUw<^*PdRc{=dfraZHZw#Z0tlWSnY@Sa^b&?#100@ z{_FfRQLnB@!T!=@b~^OQ%=kOqZV~#G$Z_lsyCBAIX#}XqC0JJtN`+VcTCgnwL#Wxv z_7{gBXE;^eNSopw^FE}{F|I6jv(=ml{Zi3BrW(Pignjigj+33`2+gOipIKUx5*A@l}3w)qu(^K;kf=967kK*?Lhk zP5Z}7g{sOZovKXH{avzE^c}Z+kZKUGI|;Ww0Z#o>c+2J#fnBUNjH%YV2AoDmsuhfh z#+|_`vH>f?@U^eo8)4oC`x-%(i?Zd2)zeO>803Q7pXKIRh^!~ zg}a3Dl<#z1zFdqjm~+oD;p^!rR+Rgl+&;R^9OLOX{+}jaPYd3R|lSI8*v*$K^tNnfyB&*v!x(M@$;2V0eOu6z~Cf)v)A;AD`w36U_rY-$3cG~(T3l;r`C7#0O9C? z$M`RFc@PO@A=^-KOxt2YrI|Fvl$fJ^`{A(}SlAGR!D7Amxx3UP^V@ifgCJXLlxVYa z(G?H#E;a1-@c(A5>hKB-tIi5?^QN#PZ2N!G|G}wip%-&tGVV9UWU;kXUB`^UdkF`A z?C*|8X?#vUk~@-i<^M@jTQvUP0c$y|39a%QX3+9_Ysdl*T?PU7Em5gA4kyvo$aCQ^ z7-P2w!;&bdnaufruukem$@|0M-2L{jn>YXxAz6{$ri@ZhIru%%f6b*UR%L&=NFM;-BfA0*>&p6KD$lZ9>GC@3(yayiWLgIDQ}>15%^b^kc#=3dw4Y)lNF7gQ&a5||J0EYO|w^%}v6Q75u|AO z?Dfdr{1)kLCRz)a)8Y@rZ*)qISs%5o?8DP?6e|>eB#B_WiNe8yg)z)}O&J;}CZ!+|*~~yc>$+n~gBIM?R(=Db}NldsHgE)w34Ah=7wXEU(SAm5_FEF!Ej3 zzT;vkE*Q^!e;|)MhZY4n3eL5@94=7XAuGNx@t!uu*O{2xBxBM(jfunH($-{>S!6q<%^3Jh~K? z|5&vff=f}5Wt`Uj32-}OsIA6`V}@emCk41&eAEqXMs&D>gI}JQSg)abg+aDXt+R`K zEHXAuSBV%62`2GY`Bk%HRdS%HceY7QPX@voOfNY6j`E%$tS426}u4 zJDhV|;CCA_>lm$ATPef5FqN6Gz%>a2qQ36azoPE^We^Gt$UJvsOvTsAkipe=&tcN- zFx>8|b)5kvi>bNEa`Bgr@K#VWz)~>p1;K!=Z=~S|XVT+1@M_#^_szeQl#Y&n_bVXF zDjyux=ju#=@gm|W> zqo?JwhpCt2cM9$o=(a`*;Jz6vq$S^2duO85Waa!%KpV>7Nj)SNqv((ccA&B48!obJ zL+qw;tQF9tE$I8~-Zr>an0qa(nEnd`F?8_srbHT!wMvV{ayQUDPgXv);-i)D0e(y; zq{erlKl>VZkpcd=>#<36I%ihJ{4RN9e+M*6;~t!oOxQZ2pNMOACNv%+f9h%KO%8TRZsG2w}Q_j6Pnx5FaeHAw4 zXksQf^$6v`;OsfY?@}BSsowGAZhK%EU=CT`rd{A(yJL-EI*bsBtd8?P?*;8A%9J@u z8AH6VI&R^ACGQk*qiL6y^rm<1f&dfiR1?Z7%^)YfVyCF=>A+t=Shv{SI)oYcBMa=(i_k_jEgik z$ZK$1&tmKq=_Y^$xvqEip%|rF)ebU>Vci3jdoKA zyg-y%wbmk`;!F}!acvJFx8Wjw81eMRy3jniUa{urq(-CMw!IHnSFfUavL%ccv9{ze zpRV8m_5Uz{b~XQbi*0^J4TSE9La(!QCIkQ`=5e*NdoVylOh2sxXL&pVnd}&E39k=6 zA5Phy>%@!~Fpg55h7;$c=ZJYyJC^v9Han5U+2?UGfMz?Jk$F=hm0+yDxD){fP|q!; z>9e}GH6Q&|n{-WQxWtwCc`in0Pt>iUq@Fj5jvV#Hft(KRBLnlvjrn8Y$kM{_tLL&Q zS|FowFfaa}5paMb?CkLql^;79pn*-goHTBDYpMGM(772qc~3_4&qI}tRaZ}-8JSX6 zHMdI1t!%%WebwZM1 zh3w@jX{tp~BNsuPAsJc&-52e18`VM>WKybTP6|E$K>fHapYIvBact-6%#!)C2b_s| zZDJhQMWe&JR#JWP$Xm(8c|S|gP52naWK>G5FtatqjFtCWk8QDwr zjuLWMS2&HgXPC}G2Mr0X%9oYsX_0i&j%qFI5u zMnviEAv^p}mA&ybGfG6EU{d%DKF-WP(L=@h2DQ(S8hSFuE-$ zDmJagOQrwZmwJHHQNBWgSB@OZz8fY_&jC2$y^|c@40|c0?_-NQ|JV4l_sDFF58E+9 z3w_r$ImGQ(1#b7RituyC^1A(k40Vau7{B@^lS(pcX~}rh&iqlP>p{l|(cgSTv4jwU zsznvr6ydGwcsf;oHhR-Rc~xc^9c(qCg6K@5`L3u92IbpiPjmmMMN7};38=4qkSqSb zt+yFj!#sD^%gj-AkGksqm7C*J;$~d9D%*xOj3Y0_qTNfaZ^&b<)g`U*Dm#EKtMk&3 z0n5fI$1F^p*t=NX?$$)1pTGXfwNfXM!IM4T>flHtH$5DHIKL>36om6F_61svg7ViU zBCGT5AmmMB===Xly;S(I5ON9pt35sUVkqguT2~h0Z~s+U6Jan(z~J9`a8X%`r}6&> f|L-+egd%)`A!8W^F01|1^8xbGDpK_lX5s$_bX~*Z literal 0 HcmV?d00001 diff --git a/frontend/src/models/dataConnector.js b/frontend/src/models/dataConnector.js index b35b5ff90..ec5d5b90a 100644 --- a/frontend/src/models/dataConnector.js +++ b/frontend/src/models/dataConnector.js @@ -162,6 +162,29 @@ const DataConnector = { }); }, }, + + drupalwiki: { + collect: async function ({ baseUrl, spaceIds, accessToken }) { + return await fetch(`${API_BASE}/ext/drupalwiki`, { + method: "POST", + headers: baseHeaders(), + body: JSON.stringify({ + baseUrl, + spaceIds, + accessToken, + }), + }) + .then((res) => res.json()) + .then((res) => { + if (!res.success) throw new Error(res.reason); + return { data: res.data, error: null }; + }) + .catch((e) => { + console.error(e); + return { data: null, error: e.message }; + }); + }, + }, }; export default DataConnector; diff --git a/server/endpoints/extensions/index.js b/server/endpoints/extensions/index.js index 8f836ce07..7bfff0672 100644 --- a/server/endpoints/extensions/index.js +++ b/server/endpoints/extensions/index.js @@ -127,6 +127,27 @@ function extensionEndpoints(app) { } } ); + app.post( + "/ext/drupalwiki", + [validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])], + async (request, response) => { + try { + const responseFromProcessor = + await new CollectorApi().forwardExtensionRequest({ + endpoint: "/ext/drupalwiki", + method: "POST", + body: request.body, + }); + await Telemetry.sendTelemetry("extension_invoked", { + type: "drupalwiki", + }); + response.status(200).json(responseFromProcessor); + } catch (e) { + console.error(e); + response.sendStatus(500).end(); + } + } + ); } module.exports = { extensionEndpoints }; diff --git a/server/jobs/sync-watched-documents.js b/server/jobs/sync-watched-documents.js index 43dbf7515..0b3a72d1d 100644 --- a/server/jobs/sync-watched-documents.js +++ b/server/jobs/sync-watched-documents.js @@ -34,7 +34,7 @@ const { DocumentSyncRun } = require('../models/documentSyncRun.js'); continue; } - if (type === 'link' || type === 'youtube') { + if (['link', 'youtube'].includes(type)) { const response = await collector.forwardExtensionRequest({ endpoint: "/ext/resync-source-document", method: "POST", @@ -46,7 +46,7 @@ const { DocumentSyncRun } = require('../models/documentSyncRun.js'); newContent = response?.content; } - if (type === 'confluence' || type === 'github' || type === 'gitlab') { + if (['confluence', 'github', 'gitlab', 'drupalwiki'].includes(type)) { const response = await collector.forwardExtensionRequest({ endpoint: "/ext/resync-source-document", method: "POST", diff --git a/server/models/documentSyncQueue.js b/server/models/documentSyncQueue.js index 860a67018..4c5ee71af 100644 --- a/server/models/documentSyncQueue.js +++ b/server/models/documentSyncQueue.js @@ -10,7 +10,14 @@ const { Telemetry } = require("./telemetry"); const DocumentSyncQueue = { featureKey: "experimental_live_file_sync", // update the validFileTypes and .canWatch properties when adding elements here. - validFileTypes: ["link", "youtube", "confluence", "github", "gitlab"], + validFileTypes: [ + "link", + "youtube", + "confluence", + "github", + "gitlab", + "drupalwiki", + ], defaultStaleAfter: 604800000, maxRepeatFailures: 5, // How many times a run can fail in a row before pruning. writable: [], @@ -52,6 +59,7 @@ const DocumentSyncQueue = { if (chunkSource.startsWith("confluence://")) return true; // If is a confluence document link if (chunkSource.startsWith("github://")) return true; // If is a GitHub file reference if (chunkSource.startsWith("gitlab://")) return true; // If is a GitLab file reference + if (chunkSource.startsWith("drupalwiki://")) return true; // If is a DrupalWiki document link return false; },