From e1c350f7134e8f58125ba85e94f791aeb96d06bb Mon Sep 17 00:00:00 2001 From: David Krause Date: Tue, 14 Feb 2023 17:13:18 +0100 Subject: [PATCH 1/4] Added robots.txt and periodically sitemap creation, for better SEO. Signed-off-by: David Krause --- .gitignore | 2 ++ nimforum.nimble | 4 ++-- src/forum.nim | 16 ++++++++++++++-- src/setup_nimforum.nim | 3 +++ src/sitemap.nim | 30 ++++++++++++++++++++++++++++++ 5 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 src/sitemap.nim diff --git a/.gitignore b/.gitignore index fe26a8a8..fe013d9c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ *.swp nimcache/ *.db* +public/sitemap*.xml +public/robots.txt # Specific paths /createdb diff --git a/nimforum.nimble b/nimforum.nimble index 68eb8c02..d471ea9d 100644 --- a/nimforum.nimble +++ b/nimforum.nimble @@ -1,5 +1,5 @@ # Package -version = "2.2.0" +version = "2.3.0" author = "Dominik Picheta" description = "The Nim forum" license = "MIT" @@ -23,7 +23,7 @@ requires "sass#649e0701fa5c" requires "karax#45bac6b" requires "webdriver#c5e4182" - +requires "https://github.com/enthus1ast/nimSimpleSitemap.git#d64c4a6" when NimMajor >= 1 and NimMinor >= 9: requires "db_connector >= 0.1.0" requires "smtp >= 0.1.0" diff --git a/src/forum.nim b/src/forum.nim index 1da96161..d477fc23 100644 --- a/src/forum.nim +++ b/src/forum.nim @@ -14,7 +14,7 @@ import import cgi except setCookie import std/options -import auth, email, utils, buildcss +import auth, email, utils, buildcss, sitemap when NimMajor >= 1 and NimMinor >= 9: import db_connector/db_sqlite @@ -50,6 +50,7 @@ var config: Config captcha: ReCaptcha mailer: Mailer + sitemapGenerator: SitemapGenerator karaxHtml: string proc init(c: TForumData) = @@ -253,6 +254,13 @@ proc verifyIdentHash( if newIdent != ident: raise newForumError("Invalid ident hash") +proc periodicallyCreateSitemap() {.async.} = + while true: + info "Regenerate sitemap" + sitemapGenerator.generate() + await sleepAsync(1000 * 60 * 60 * 24) # One day + + proc initialise() = randomize() @@ -270,6 +278,10 @@ proc initialise() = isFTSAvailable = db.getAllRows(sql("SELECT name FROM sqlite_master WHERE " & "type='table' AND name='post_fts'")).len == 1 + sitemapGenerator = newSitemapGenerator(db, config.hostname) + info "Generate sitemap on start" + sitemapGenerator.generate() + asyncCheck periodicallyCreateSitemap() buildCSS(config) # Read karax.html and set its properties. @@ -802,7 +814,7 @@ proc updateProfile( if c.rank < Moderator and wasEmailChanged: if rank != EmailUnconfirmed: raise newForumError("Rank needs a change when setting new email.") - + {.cast(gcsafe).}: await sendSecureEmail( mailer, ActivateEmail, c.req, row[0], row[1], email, row[3] diff --git a/src/setup_nimforum.nim b/src/setup_nimforum.nim index 0485b7f2..9be810e3 100644 --- a/src/setup_nimforum.nim +++ b/src/setup_nimforum.nim @@ -320,6 +320,9 @@ These can be changed later in the generated forum.json file. dbPath ) + echo("Creating public/robots.txt") + writeFile(getAppDir() / "public", "Sitemap: " & hostname & "sitemap.xml") + echo("Setup complete!") proc echoHelp() = diff --git a/src/sitemap.nim b/src/sitemap.nim new file mode 100644 index 00000000..2b337a58 --- /dev/null +++ b/src/sitemap.nim @@ -0,0 +1,30 @@ +import simpleSitemap, db_sqlite, times, os, uri + +type + SitemapGenerator* = object + dbconn: Dbconn + hostname: Uri + +proc newSitemapGenerator*(dbconn: DbConn, hostname: string): SitemapGenerator = + ## The generator for sitemaps `dbconn` is the forums database, `hostname` is the absolute url + ## the forum runs on. + result = SitemapGenerator() + result.dbconn = dbconn + result.hostname = parseUri(hostname) + +proc generate*(sg: SitemapGenerator) = + ## Generates the xml sitemap(s) creates them in `getAppDir() / "public"` + var urlDates: seq[UrlDate] = @[] + # for row in sg.dbconn.rows(sql"select id, name, modified from thread"): + # let absurl = sg.hostname / "t" / row[0] / row[1] + for row in sg.dbconn.rows(sql"select id, modified from thread"): + let absurl = sg.hostname / "t" / row[0] + let pdate = row[1].parse("yyyy-MM-dd H':'m':'s") + urlDates.add ($absurl, pdate) + let pages = generateSitemaps( + urlDates, + urlsOnRecent = 30, + maxUrlsPerSitemap = 50_000, + base = $sg.hostname + ) + write(pages, folder = getAppDir() / "public") \ No newline at end of file From ab0de12f414bd01904a35bfa4a2fa5232b9b9e1e Mon Sep 17 00:00:00 2001 From: David Krause Date: Tue, 14 Feb 2023 17:19:12 +0100 Subject: [PATCH 2/4] Signed-off-by: David Krause --- src/sitemap.nim | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/sitemap.nim b/src/sitemap.nim index 2b337a58..d7c1c977 100644 --- a/src/sitemap.nim +++ b/src/sitemap.nim @@ -15,8 +15,6 @@ proc newSitemapGenerator*(dbconn: DbConn, hostname: string): SitemapGenerator = proc generate*(sg: SitemapGenerator) = ## Generates the xml sitemap(s) creates them in `getAppDir() / "public"` var urlDates: seq[UrlDate] = @[] - # for row in sg.dbconn.rows(sql"select id, name, modified from thread"): - # let absurl = sg.hostname / "t" / row[0] / row[1] for row in sg.dbconn.rows(sql"select id, modified from thread"): let absurl = sg.hostname / "t" / row[0] let pdate = row[1].parse("yyyy-MM-dd H':'m':'s") From bde07231cdbfa1e0fb1cb8ff80180b88cb7c3078 Mon Sep 17 00:00:00 2001 From: David Krause Date: Tue, 14 Feb 2023 17:40:25 +0100 Subject: [PATCH 3/4] fix robots.txt generation Signed-off-by: David Krause --- nimforum.nimble | 2 +- src/setup_nimforum.nim | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nimforum.nimble b/nimforum.nimble index d471ea9d..11f96cc1 100644 --- a/nimforum.nimble +++ b/nimforum.nimble @@ -23,7 +23,7 @@ requires "sass#649e0701fa5c" requires "karax#45bac6b" requires "webdriver#c5e4182" -requires "https://github.com/enthus1ast/nimSimpleSitemap.git#d64c4a6" +requires "https://github.com/enthus1ast/nimSimpleSitemap.git#f62730a" when NimMajor >= 1 and NimMinor >= 9: requires "db_connector >= 0.1.0" requires "smtp >= 0.1.0" diff --git a/src/setup_nimforum.nim b/src/setup_nimforum.nim index 9be810e3..36305415 100644 --- a/src/setup_nimforum.nim +++ b/src/setup_nimforum.nim @@ -7,7 +7,7 @@ # # Script to initialise the nimforum. -import strutils, os, times, json, options, terminal +import strutils, os, times, json, options, terminal, uri when NimMajor >= 1 and NimMinor >= 9: import db_connector/db_sqlite @@ -321,7 +321,7 @@ These can be changed later in the generated forum.json file. ) echo("Creating public/robots.txt") - writeFile(getAppDir() / "public", "Sitemap: " & hostname & "sitemap.xml") + writeFile("public" / "robots.txt", "Sitemap: " & $(parseUri(hostname) / "sitemap.xml")) echo("Setup complete!") From 08522626cae9711df30dfd0eff596f6ed7a3708d Mon Sep 17 00:00:00 2001 From: David Krause Date: Fri, 24 Feb 2023 09:29:17 +0100 Subject: [PATCH 4/4] fix db_sqlite import Signed-off-by: David Krause --- src/sitemap.nim | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/sitemap.nim b/src/sitemap.nim index d7c1c977..996f358b 100644 --- a/src/sitemap.nim +++ b/src/sitemap.nim @@ -1,4 +1,9 @@ -import simpleSitemap, db_sqlite, times, os, uri +import simpleSitemap, times, os, uri + +when NimMajor >= 1 and NimMinor >= 9: + import db_connector/db_sqlite +else: + import std/db_sqlite type SitemapGenerator* = object