From d4d47e665fe8b2085dc5c2d60fd245661fa5fc24 Mon Sep 17 00:00:00 2001 From: matthew Date: Fri, 7 Oct 2022 19:12:31 +0400 Subject: [PATCH] add site counter --- searching-front/package-lock.json | 25 ++++ searching-front/package.json | 3 + searching-front/services/domain-watcher.ts | 125 ++++++++++-------- searching-front/services/influx.ts | 14 ++ searching-front/services/main.ts | 29 ++-- .../services/modules/influxdb/constants.ts | 5 + .../services/modules/influxdb/helpers.ts | 46 +++++++ .../services/modules/influxdb/index.ts | 52 ++++++++ .../services/modules/influxdb/types.ts | 13 ++ searching-front/services/parser.ts | 2 +- 10 files changed, 242 insertions(+), 72 deletions(-) create mode 100644 searching-front/services/influx.ts create mode 100644 searching-front/services/modules/influxdb/constants.ts create mode 100644 searching-front/services/modules/influxdb/helpers.ts create mode 100644 searching-front/services/modules/influxdb/index.ts create mode 100644 searching-front/services/modules/influxdb/types.ts diff --git a/searching-front/package-lock.json b/searching-front/package-lock.json index e1cf91c..4d0ce51 100644 --- a/searching-front/package-lock.json +++ b/searching-front/package-lock.json @@ -13,6 +13,8 @@ "@blitzjs/rpc": "2.0.0-beta.3", "@elastic/elasticsearch": "8.2.1", "@hookform/resolvers": "2.9.7", + "@influxdata/influxdb-client": "1.31.0", + "@influxdata/influxdb-client-apis": "1.31.0", "@prisma/client": "4.2.1", "@types/html-to-text": "8.1.1", "@types/textversionjs": "1.1.1", @@ -2198,6 +2200,19 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "node_modules/@influxdata/influxdb-client": { + "version": "1.31.0", + "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client/-/influxdb-client-1.31.0.tgz", + "integrity": "sha512-8DVT3ZB/VeCK5Nn+BxhgMrAMSTseQAEgV20AK+ZMO5Fcup9XWsA9L2zE+3eBFl0Y+lF3UeKiASkiKMQvws35GA==" + }, + "node_modules/@influxdata/influxdb-client-apis": { + "version": "1.31.0", + "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client-apis/-/influxdb-client-apis-1.31.0.tgz", + "integrity": "sha512-6ALGNLxtfffhICobOdj13Z6vj6gdQVOzVXPoPNd+w7V60zrbGhTqzXHV1KMZ/lzOb6YkRTRODbxz4W/b/7N5hg==", + "peerDependencies": { + "@influxdata/influxdb-client": "*" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", @@ -18167,6 +18182,16 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "@influxdata/influxdb-client": { + "version": "1.31.0", + "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client/-/influxdb-client-1.31.0.tgz", + "integrity": "sha512-8DVT3ZB/VeCK5Nn+BxhgMrAMSTseQAEgV20AK+ZMO5Fcup9XWsA9L2zE+3eBFl0Y+lF3UeKiASkiKMQvws35GA==" + }, + "@influxdata/influxdb-client-apis": { + "version": "1.31.0", + "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client-apis/-/influxdb-client-apis-1.31.0.tgz", + "integrity": "sha512-6ALGNLxtfffhICobOdj13Z6vj6gdQVOzVXPoPNd+w7V60zrbGhTqzXHV1KMZ/lzOb6YkRTRODbxz4W/b/7N5hg==" + }, "@jridgewell/gen-mapping": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", diff --git a/searching-front/package.json b/searching-front/package.json index 5228314..043009f 100644 --- a/searching-front/package.json +++ b/searching-front/package.json @@ -3,6 +3,7 @@ "version": "1.0.0", "scripts": { "watcher": "ts-node-esm ./services/main.ts", + "influx": "ts-node-esm ./services/influx.ts", "parser": "ts-node-esm ./services/parser.ts", "dev": "blitz dev", "build": "blitz build", @@ -30,6 +31,8 @@ "@blitzjs/rpc": "2.0.0-beta.3", "@elastic/elasticsearch": "8.2.1", "@hookform/resolvers": "2.9.7", + "@influxdata/influxdb-client": "1.31.0", + "@influxdata/influxdb-client-apis": "1.31.0", "@prisma/client": "4.2.1", "@types/html-to-text": "8.1.1", "@types/textversionjs": "1.1.1", diff --git a/searching-front/services/domain-watcher.ts b/searching-front/services/domain-watcher.ts index f1c7202..e4f7055 100644 --- a/searching-front/services/domain-watcher.ts +++ b/searching-front/services/domain-watcher.ts @@ -1,4 +1,3 @@ - import tonweb from "tonweb" import { JettonApi, @@ -14,83 +13,93 @@ import db from "../db/index" import axios from "axios" import { getTonProxy } from "./helpers" - interface SearchNFTItemsParams { limit: number offset: number } -const wait = (time:number) => new Promise((resolve)=>setTimeout(()=>resolve(true),time )) +const getFullUrl = (dmn: string) => `http://${dmn}` +const upsertDmn = async (dmn: string, available: boolean) => + await db.nftDomain.upsert({ + where: { + address: getFullUrl(dmn), + }, + update: { available, address: getFullUrl(dmn) }, + create: { available, address: getFullUrl(dmn) }, + }) -const searchNFTItems = async ({ limit, offset }: SearchNFTItemsParams) => { - try{ +const wait = (time: number) => new Promise((resolve) => setTimeout(() => resolve(true), time)) - - console.log(`Start search limit:${limit}, offset:${offset}`) - await wait(1000) - const { data } = await axios.get( - `https://tonapi.io/v1/nft/searchItems?collection=EQC3dNlesgVD8YbAazcauIrXBPfiVhMMr5YYk2in0Mtsz0Bz&include_on_sale=false&limit=${limit}&offset=${offset}`, - { - headers:{ - // 'Authorization': 'Bearer '+ '6c456b1e31217a79e121dcb9b506c280358d58bc86659bdbac1d737bfc3691fb', +const searchNFTItems = async ({ limit, offset }: SearchNFTItemsParams) => { + try { + console.log(`Start search limit:${limit}, offset:${offset}`) + await wait(1000) + const { data } = await axios.get( + `https://tonapi.io/v1/nft/searchItems?collection=EQC3dNlesgVD8YbAazcauIrXBPfiVhMMr5YYk2in0Mtsz0Bz&include_on_sale=false&limit=${limit}&offset=${offset}`, + { + headers: { + // 'Authorization': 'Bearer '+ '6c456b1e31217a79e121dcb9b506c280358d58bc86659bdbac1d737bfc3691fb', + }, } - } - ) - return data.nft_items -} catch (e){ - return searchNFTItems({ limit, offset }) -} + ) + return data.nft_items + } catch (e) { + return searchNFTItems({ limit, offset }) + } } const portion = 1000 const fetchTonSite = async (url: string) => { - const urlToFetch = `http://${url}/` - const response = await axios.get(urlToFetch, { - proxy: getTonProxy(), - }) - if (!response.data) { - console.log("Error fetch") - throw "error" + try { + const urlToFetch = `http://${url}/` + const response = await axios.get(urlToFetch, { + proxy: getTonProxy(), + }) + if (!response.data) { + console.log("Error fetch") + throw "err" + } + return url + } catch (e) { + throw url } - return url } -const main = async () => new Promise(async (resolve)=>{ - // Receive typed array of owner nfts - let count = 0 - while (true) { - // в nftItems 1000 сайтов - const nftItems = await searchNFTItems({ - limit: portion, - offset: count * portion, - }) +const main = async () => + new Promise(async (resolve) => { + // Receive typed array of owner nfts + let count = 0 + while (true) { + // в nftItems 1000 сайтов + const nftItems = await searchNFTItems({ + limit: portion, + offset: count * portion, + }) - if (nftItems.length) { - for (let i = 0; i < nftItems.length; i++) { - const nftDomainItem = nftItems[i] - if (nftDomainItem.dns) { - fetchTonSite(nftDomainItem.dns) - .then(async (dmn) => { - console.log("success dmn", dmn) - await db.nftDomain.upsert({ - where: { - address: `http://${dmn}`, - }, - update: { available: false, address: `http://${dmn}` }, - create: { available: false, address: `http://${dmn}` }, + if (nftItems.length) { + for (let i = 0; i < nftItems.length; i++) { + const nftDomainItem = nftItems[i] + if (nftDomainItem.dns) { + fetchTonSite(nftDomainItem.dns) + .then(async (dmn) => { + console.log("success dmn", dmn) + upsertDmn(dmn, true) }) - }) - .catch(() => {}) + .catch((dmn) => { + upsertDmn(dmn,false) + }) + } } + count++ + continue } - count++ - continue + break } - break - } - console.log('Finish fetch nft') - setTimeout(()=>{resolve(true)}, 10000) -}) + console.log("Finish fetch nft") + setTimeout(() => { + resolve(true) + }, 10000) + }) export default main diff --git a/searching-front/services/influx.ts b/searching-front/services/influx.ts new file mode 100644 index 0000000..f9647ac --- /dev/null +++ b/searching-front/services/influx.ts @@ -0,0 +1,14 @@ +import influxdb from "./modules/influxdb" + +import db from "../db/index" + +const main = async () => { + const allDomainsCount = await db.nftDomain.count() + const availableDomainsCount = await db.nftDomain.count({ where: { available: true } }) + influxdb.writeSitesCount({ + all: allDomainsCount, + available: availableDomainsCount, + }) +} + +export default main diff --git a/searching-front/services/main.ts b/searching-front/services/main.ts index 03be486..a887bbb 100644 --- a/searching-front/services/main.ts +++ b/searching-front/services/main.ts @@ -1,31 +1,34 @@ import dotenv from "dotenv" import path from "path" dotenv.config({ path: path.resolve(__dirname, "../.env.local") }) -import domainWatcher from './domain-watcher' -import parser from './parser' +import domainWatcher from "./domain-watcher" +import parser from "./parser" +import influx from "./influx" const run = async()=>{ console.log('Start domain watcher') console.time('watcher') await domainWatcher(); console.timeEnd('watcher') + influx() console.log('Start parser'); console.time('watcher'); await parser(); console.timeEnd('watcher'); + } -const second = 1000; -const minute= 60 * second; -const hour = 60 * minute; +const second = 1000 +const minute = 60 * second +const hour = 60 * minute -run(); +run() -setInterval(()=>{ - console.log(new Date(), 'Health check'); -},hour) +setInterval(() => { + console.log(new Date(), "Health check") +}, hour) -setInterval(()=>{ - console.log(new Date(), 'Cron parse start') - run() -},3 * hour) \ No newline at end of file +setInterval(() => { + console.log(new Date(), "Cron parse start") + run() +}, 3 * hour) diff --git a/searching-front/services/modules/influxdb/constants.ts b/searching-front/services/modules/influxdb/constants.ts new file mode 100644 index 0000000..e0d60e5 --- /dev/null +++ b/searching-front/services/modules/influxdb/constants.ts @@ -0,0 +1,5 @@ +export const influxToken = process.env.INFLUX_TOKEN as string; +export const influxOrg = process.env.INFLUX_ORG as string +export const influxBucket = process.env.INFLUX_BUCKET as string +export const influxPointName = 'count'; +export const influxHost = 'Searching'; diff --git a/searching-front/services/modules/influxdb/helpers.ts b/searching-front/services/modules/influxdb/helpers.ts new file mode 100644 index 0000000..671d520 --- /dev/null +++ b/searching-front/services/modules/influxdb/helpers.ts @@ -0,0 +1,46 @@ +import { fluxDuration } from "@influxdata/influxdb-client"; +import { influxBucket, influxHost, influxPointName } from "./constants"; +import { InfluxField, InfluxPeriod } from "./types"; + +export const influxQuery = (field: InfluxField, fetchPeriod: InfluxPeriod) =>{ + let start; + let period; + + switch(fetchPeriod){ + case InfluxPeriod.H: + start ='-1h' + period = '6m' + case InfluxPeriod.D: + start ='-1d' + period = '144m' + case InfluxPeriod.W: + start ='-1w' + period = '1008m' + case InfluxPeriod.M: + start ='-1mo' + period = '3d' + case InfluxPeriod.Y: + start ='-1y' + period = '36d' + case InfluxPeriod.tenminute: + start ='10m' + period = '1m' + } + + const influxPeriod = fluxDuration(period); + const influxStart = fluxDuration(start); + return `from(bucket: "${influxBucket}") + |> range(start: ${start}, stop: now()) + |> filter(fn: (r) => r["host"] == "${influxHost}") + |> filter(fn: (r) => r["_field"] == "${field}") + |> filter(fn: (r) => r["_measurement"] == "${influxPointName}") + |> aggregateWindow(every: ${period}, fn: mean, createEmpty: false) + |> yield(name: "mean")` +} + +export const processInfluxResult = (res:unknown[]) => { + return res.map(i=>({ + value: i._value, + time: i._time + })) +} \ No newline at end of file diff --git a/searching-front/services/modules/influxdb/index.ts b/searching-front/services/modules/influxdb/index.ts new file mode 100644 index 0000000..d6139ce --- /dev/null +++ b/searching-front/services/modules/influxdb/index.ts @@ -0,0 +1,52 @@ +import { fluxDuration, InfluxDB } from "@influxdata/influxdb-client" +import { Point } from "@influxdata/influxdb-client" +import { influxBucket, influxHost, influxOrg, influxPointName, influxToken } from "./constants" +import { influxQuery, processInfluxResult } from "./helpers"; +import { InfluxField, InfluxPeriod } from "./types" + +interface WriteSitesCounteParams { + all: number; + available: number; +} + +interface QueryParams { + field: InfluxField; + period: InfluxPeriod; +} + +class InfluxDb { + private client: InfluxDB + constructor() { + this.client = new InfluxDB({ url: process.env.INFLUX_URL as string, token: influxToken }) + } + getWriteApi() { + const writeApi = this.client.getWriteApi(influxOrg, influxBucket) + writeApi.useDefaultTags({ host: influxHost }) + return writeApi + } + writeSitesCount({all,available}:WriteSitesCounteParams){ + const writeApi = this.getWriteApi() + const pointAll = new Point(influxPointName).intField(InfluxField.ALL_SITES,all); + const pointAvailable = new Point(influxPointName).intField(InfluxField.AVAILABLE_SITES,available); + writeApi.writePoint(pointAll) + writeApi.writePoint(pointAvailable) + writeApi.close() + } + async query({field,period}:QueryParams){ + const queryApi = this.client.getQueryApi(influxOrg) + + const query = influxQuery(field, period); + const result = await queryApi.collectRows(query) + console.log(processInfluxResult(result)); + } + getAllSiteCounts(period:InfluxPeriod){ + + this.query({ + field:InfluxField.ALL_SITES, + period + }) + + } +} + +export default new InfluxDb() diff --git a/searching-front/services/modules/influxdb/types.ts b/searching-front/services/modules/influxdb/types.ts new file mode 100644 index 0000000..2230c4c --- /dev/null +++ b/searching-front/services/modules/influxdb/types.ts @@ -0,0 +1,13 @@ +export enum InfluxPeriod { + H='H', + D='D', + W='W', + M='M', + Y='Y', + tenminute='tenminute', +} + +export enum InfluxField { + ALL_SITES = 'ALL_SITES', + AVAILABLE_SITES = 'AVAILABLE_SITES' +} \ No newline at end of file diff --git a/searching-front/services/parser.ts b/searching-front/services/parser.ts index 58a1a13..3095cca 100644 --- a/searching-front/services/parser.ts +++ b/searching-front/services/parser.ts @@ -41,7 +41,7 @@ const indexWebsite = async (domain: string, path: string, subpages: SubPages = { const main = async () => { await Elastic.initElastic() - const domains = await db.nftDomain.findMany() + const domains = await db.nftDomain.findMany({where:{available: true}}) console.log('Find domains', domains) if (domains) { for (const domain of domains) {