saberbud/WechatExporter
1
0
Fork 0
mirror of https://github.com/BlueMatthew/WechatExporter.git synced 2026-08-29 16:40:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

Optimize async loading

Browse Source
This commit is contained in:
Matthew
2021-07-23 23:10:22 +08:00
parent fe00ba4f8c
commit 14e1ce1ffd
3 changed files with 167 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
WechatExporter/core/Exporter.cpp
+42 -19
View File
@@ -812,20 +812,10 @@ int Exporter::exportSession(const Friend& user, const MessageParser& msgParser,
auto b = messages.cbegin();
// No page for text mode
auto e = (((m_options & (SPO_TEXT_MODE | SPO_SYNC_LOADING)) != 0) || (messages.size() <= pageSize)) ? messages.cend() : (b + pageSize);
std::string moreMsgs = ((m_options & (SPO_TEXT_MODE | SPO_SYNC_LOADING)) != 0) ? "[]" : "[]"; // [] = empty json array
if (e != messages.cend())
{
Json::Value jsonMsgs(Json::arrayValue);
for (auto it = e; it != messages.cend(); ++it)
{
jsonMsgs.append(*it);
}
Json::FastWriter writer;
moreMsgs = writer.write(jsonMsgs);
}
std::string scripts = (m_options & SPO_SYNC_LOADING) || (messages.size() <= pageSize) ? "" : getTemplate("scripts");
const size_t numberOfMessages = std::distance(e, messages.cend());
const size_t numberOfPages = (numberOfMessages + pageSize - 1) / pageSize;
std::string html = getTemplate("frame");
#ifndef NDEBUG
replaceAll(html, "%%USRNAME%%", user.getUsrName() + " - " + user.getHash());
@@ -835,17 +825,50 @@ int Exporter::exportSession(const Friend& user, const MessageParser& msgParser,
replaceAll(html, "%%SESSION_USRNAME%%", "");
#endif
replaceAll(html, "%%DISPLAYNAME%%", session.getDisplayName());
replaceAll(html, "%%JSONDATA%%", moreMsgs);
replaceAll(html, "%%ASYNCLOADINGTYPE%%", m_loadingDataOnScroll ? "onscroll" : "initial");
replaceAll(html, "%%ASYNC_LOADING_TYPE%%", m_loadingDataOnScroll ? "onscroll" : "initial");
replaceAll(html, "%%SIZE_OF_PAGE%%", std::to_string(pageSize));
replaceAll(html, "%%NUMBER_OF_MSGS%%", std::to_string(numberOfMessages));
replaceAll(html, "%%NUMBER_OF_PAGES%%", std::to_string(numberOfPages));
replaceAll(html, "%%DATA_PATH%%", encodeUrl(session.getOutputFileName() + "_files") + "/Data");
replaceAll(html, "%%BODY%%", join(b, e, ""));
replaceAll(html, "%%LOADING_SCRIPTS%%", scripts);
replaceAll(html, "%%HEADER_FILTER%%", (m_options & SPO_SUPPORT_FILTER) ? getTemplate("filter") : "");
std::string fileName = combinePath(outputBase, session.getOutputFileName() + "." + m_extName);
writeFile(fileName, html);
if ((m_options & SPO_SYNC_LOADING) == 0 && numberOfPages > 0)
{
std::string dataPath = combinePath(outputBase, session.getOutputFileName() + "_files", "Data");
makeDirectory(dataPath);
for (size_t page = 0; page < numberOfPages; ++page)
{
b = e;
std::string scripts = getTemplate("scripts");
e = (page == (numberOfPages - 1)) ? messages.cend() : (b + pageSize);
Json::Value jsonMsgs(Json::arrayValue);
for (auto it = b; it != e; ++it)
{
jsonMsgs.append(*it);
}
Json::StreamWriterBuilder builder;
builder["indentation"] = ""; // assume default for comments is None
#ifndef NDEBUG
builder["emitUTF8"] = true;
#endif
std::string moreMsgs = Json::writeString(builder, jsonMsgs);
replaceAll(scripts, "%%JSON_DATA%%", moreMsgs);
fileName = combinePath(dataPath, "msg-" + std::to_string(page + 1) + ".js");
writeFile(fileName, scripts);
}
}
}
return numberOfMsgs;
WechatExporter/res/templates/frame.html
+113 -68
View File
@@ -449,9 +449,10 @@
}
}
if (visibleMsgs < 1000 && (typeof window.moreWechatMsgs !== 'undefined') && (window.moreWechatMsgs.length > 0))
if (visibleMsgs < window.sizeOfMsgPage && (typeof window.wechatMsgsIndexes !== 'undefined') && (window.wechatMsgsIndexes.length > 0))
{
loadMsgsForNextPage(1000 - visibleMsgs);
window.numberOfMsgsToLoad = window.sizeOfMsgPage - visibleMsgs;
loadMsgsForNextPage();
}
document.body.style.cursor = 'default';
}
@@ -527,9 +528,10 @@
visibleMsgs++;
}
}
if (visibleMsgs < 1000 && (typeof window.moreWechatMsgs !== 'undefined') && (window.moreWechatMsgs.length > 0))
if (visibleMsgs < window.sizeOfMsgPage && (typeof window.wechatMsgsIndexes !== 'undefined') && (window.wechatMsgsIndexes.length > 0))
{
loadMsgsForNextPage(1000 - visibleMsgs);
window.numberOfMsgsToLoad = window.sizeOfMsgPage - visibleMsgs;
loadMsgsForNextPage();
}
document.body.style.cursor = 'default';
@@ -548,60 +550,73 @@
{
showElements("video");
}
function loadMsgsForNextPage(numberOfMsgsToLoad)
function loadMsgsForNextPage()
{
if ((typeof window.moreWechatMsgs === 'undefined') || (window.moreWechatMsgs.length == 0))
console.log("loadMsgsForNextPage starts...");
if ((typeof window.wechatMsgsIndexes === 'undefined') || (window.wechatMsgsIndexes.length == 0))
{
return 0;
}
if (typeof numberOfMsgsToLoad === 'undefined')
if ((typeof window.moreWechatMsgs === 'undefined'))
{
numberOfMsgsToLoad = 1000;
window.moreWechatMsgs = [];
}
var fragment = document.createDocumentFragment();
var div = document.createElement('div');
fragment.appendChild(div);
var visibleMsgs = 0;
while (window.moreWechatMsgs.length > 0)
var visibleMsgs = 0;
if (window.moreWechatMsgs.length > 0)
{
div.innerHTML = window.moreWechatMsgs.shift();
if (div.children.length > 0)
{
var msgDiv = div.children[0];
fragment.appendChild(msgDiv);
if (filterMsg(msgDiv))
{
visibleMsgs++;
}
}
var fragment = document.createDocumentFragment();
var div = document.createElement('div');
fragment.appendChild(div);
if (visibleMsgs >= numberOfMsgsToLoad)
while (window.moreWechatMsgs.length > 0)
{
break;
div.innerHTML = window.moreWechatMsgs.shift();
if (div.children.length > 0)
{
var msgDiv = div.children[0];
fragment.appendChild(msgDiv);
if (filterMsg(msgDiv))
{
visibleMsgs++;
}
}
}
fragment.removeChild(div);
var containerDiv = document.getElementById('msgs-div');
if (null != containerDiv)
{
containerDiv.appendChild(fragment);
}
}
fragment.removeChild(div);
var containerDiv = document.getElementById('msgs-div');
if (null != containerDiv)
window.numberOfMsgsToLoad -= visibleMsgs;
if ((window.numberOfMsgsToLoad > 0) && window.wechatMsgsIndexes.length > 0)
{
containerDiv.appendChild(fragment);
// Load next page
var nextPage = window.wechatMsgsIndexes.shift();
console.log("load next page:" + nextPage);
var script = document.createElement("script");
script.type = "text/javascript";
script.src = "%%DATA_PATH%%/msg-" + nextPage + ".js";
document.body.appendChild(script);
}
else
{
console.log("(visibleMsgs(" + visibleMsgs + ") < window.numberOfMsgsToLoad("
+ window.numberOfMsgsToLoad + ") ) && window.wechatMsgsIndexes.length(" + window.wechatMsgsIndexes.length + ") > 0");
}
return true;
}
function loadMoreMsgs()
{
window.requestAnimationFrame(function() {
if (loadMsgsForNextPage())
{
loadMoreMsgs();
}
});
}
function checkScrollDirectionIsUp(e)
{
if (e.wheelDelta)
@@ -632,38 +647,68 @@
</body>
<script language="javascript">
addEventListener('load', function (e)
(function() {
window.sizeOfMsgPage = parseInt('%%SIZE_OF_PAGE%%') || 100;
var numberOfMsgs = parseInt('%%NUMBER_OF_MSGS%%') || 0;
var numberOfPages = parseInt('%%NUMBER_OF_PAGES%%') || 0;
var asyncLoadingType = "%%ASYNC_LOADING_TYPE%%";
if (numberOfPages == 0)
{
window.moreWechatMsgs = %%JSONDATA%%;
window.currentIndex = 0;
var asyncLoadingType = "%%ASYNCLOADINGTYPE%%";
if (asyncLoadingType == "initial")
return;
}
window.wechatMsgsIndexes = [];
for (var idx = 1; idx <= numberOfPages; idx++)
{
window.wechatMsgsIndexes.push(idx);
}
console.log(window.wechatMsgsIndexes);
window.numberOfMsgsToLoad = 0;
if (asyncLoadingType == "initial")
{
// Load all messages
window.numberOfMsgsToLoad = numberOfMsgs;
window.numberOfVisibleMsgsToLoad = -1;
console.log("initial");
window.addEventListener('load', function (e)
{
window.setTimeout(loadMoreMsgs, 1000);
}
else if (asyncLoadingType == "onscroll")
console.log("onload");
loadMsgsForNextPage();
}, false);
}
else if (asyncLoadingType == "onscroll")
{
window.addEventListener('scroll', function(e)
{
document.addEventListener('scroll', function(e)
if (window.wechatMsgsIndexes.length == 0)
{
if (!checkScrollDirectionIsUp(e))
return;
}
if (!checkScrollDirectionIsUp(e))
{
var lastDiv = document.querySelector("#msgs-div > div:last-child");
var lastDivOffset = lastDiv.offsetTop + lastDiv.clientHeight;
var pageOffset = window.pageYOffset + window.innerHeight;
if(pageOffset > lastDivOffset - 20)
{
var lastDiv = document.querySelector("#msgs-div > div:last-child");
var lastDivOffset = lastDiv.offsetTop + lastDiv.clientHeight;
var pageOffset = window.pageYOffset + window.innerHeight;
if(pageOffset > lastDivOffset - 20)
{
loadMsgsForNextPage();
}
window.numberOfMsgsToLoad += window.sizeOfMsgPage;
loadMsgsForNextPage();
}
}, false);
}
}, false);
}
}, {passive: true});
}
})();
</script>
<!-- scripts.html -->
%%LOADING_SCRIPTS%%
</html>
WechatExporter/res/templates/scripts.html
+12 -3
View File
@@ -1,5 +1,14 @@
<script language="javascript">
(function() {
var msgArray = %%JSON_DATA%%;
for (var idx = 0; idx < msgArray.length; idx++)
{
window.moreWechatMsgs.push(msgArray[idx]);
}
console.log("data loaded" + msgArray.length);
msgArray = null;
loadMsgsForNextPage();
})();
</script>