Searching.ton
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

61 lines
1.7 KiB

2 years ago
import db from "../db/index"
import Elastic from "./modules/elastic"
import Parser from "./modules/parser"
import { SHOULD_NOT_PARSE } from "./modules/parser/helpers"
2 years ago
import { Domain } from "domain"
2 years ago
type SubPages = Record<string, boolean>
const findFirstNotIndexed = (subpages: SubPages = {}) => {
return Object.entries(subpages).find(([url, isIndexed]) => !isIndexed)?.[0]
}
2 years ago
const indexWebsite = async (domain: string, path: string, subpages: SubPages = {}) => {
2 years ago
const subpagesLength = Object.keys(subpages).length;
2 years ago
if (!subpages[path] && subpagesLength < 50) {
2 years ago
const url = domain + path;
const parseInfo = await Parser.parseUrl(url)
subpages[path] = true
2 years ago
let pages = {}
if (parseInfo !== SHOULD_NOT_PARSE) {
await Elastic.index(parseInfo.elasticData)
pages = {
...parseInfo.subPages,
...subpages,
}
} else {
pages = subpages
}
const firstNotIndexed = findFirstNotIndexed(pages)
if (firstNotIndexed) {
return await indexWebsite(domain, firstNotIndexed, pages)
}
} else {
const firstNotIndexed = findFirstNotIndexed(subpages)
if (firstNotIndexed) {
return await indexWebsite(domain, firstNotIndexed, subpages)
}
}
}
2 years ago
const main = async () => {
await Elastic.initElastic()
const domains = await db.nftDomain.findMany()
console.log('Find domains', domains)
2 years ago
if (domains) {
for (const domain of domains) {
2 years ago
await db.nftDomain.update({
2 years ago
where: { address: domain.address },
data: { lastParse: new Date() },
})
2 years ago
console.log('Start index domain: ',domain.address)
2 years ago
await indexWebsite(domain.address, "/")
}
}
2 years ago
console.log('Finish parse domains')
2 years ago
}
2 years ago
export default main