fix(marketing): RFC 9309 robots.txt semantics + lazy marketing import
- Remove redundant named User-agent blocks (Googlebot, Bingbot, ClaudeBot, GPTBot, PerplexityBot, Applebot) that per RFC 9309 §2.2 overrode the wildcard and granted those bots access to /api/, /admin, /account. - Add explicit Google-Extended and ChatGPT-User blocks (AI opt-in signaling) with full Allow/Disallow rule sets. - Fix /blog → /blog/ for prefix-match consistency. - Move src.marketing.routes import inside recordings.index() function to localize cross-blueprint dependency (was at module top, inverting initialization order). - Add shadow-warning comment at marketing_bp registration site.
This commit is contained in:
@@ -39,7 +39,6 @@ from src.file_exporter import export_recording, mark_export_as_deleted
|
|||||||
from src.utils.ffprobe import get_codec_info, get_creation_date, FFProbeError
|
from src.utils.ffprobe import get_codec_info, get_creation_date, FFProbeError
|
||||||
from src.utils.audio_conversion import convert_if_needed
|
from src.utils.audio_conversion import convert_if_needed
|
||||||
from src.utils.file_hash import compute_file_sha256
|
from src.utils.file_hash import compute_file_sha256
|
||||||
from src.marketing.routes import landing as _marketing_landing
|
|
||||||
|
|
||||||
# Create blueprint
|
# Create blueprint
|
||||||
recordings_bp = Blueprint('recordings', __name__)
|
recordings_bp = Blueprint('recordings', __name__)
|
||||||
@@ -1374,8 +1373,14 @@ def index():
|
|||||||
endpoints are mounted at "/". Since recordings_bp is registered first,
|
endpoints are mounted at "/". Since recordings_bp is registered first,
|
||||||
Flask's URL map resolves "/" to recordings.index, so a redirect would
|
Flask's URL map resolves "/" to recordings.index, so a redirect would
|
||||||
loop back into this same handler indefinitely.
|
loop back into this same handler indefinitely.
|
||||||
|
|
||||||
|
The src.marketing.routes import is lazy (inside the function) on
|
||||||
|
purpose: it localizes the cross-blueprint dependency to the call
|
||||||
|
site rather than coupling recordings_bp module load to marketing_bp
|
||||||
|
module load, preserving the apparent initialization order in app.py.
|
||||||
"""
|
"""
|
||||||
if not current_user.is_authenticated:
|
if not current_user.is_authenticated:
|
||||||
|
from src.marketing.routes import landing as _marketing_landing
|
||||||
return _marketing_landing()
|
return _marketing_landing()
|
||||||
|
|
||||||
# Check if user is a group admin
|
# Check if user is a group admin
|
||||||
|
|||||||
@@ -641,6 +641,8 @@ app.register_blueprint(docs_bp)
|
|||||||
# - marketing_bp at "/" (placeholder; coexists with recordings_bp.index, resolved in B-1.3)
|
# - marketing_bp at "/" (placeholder; coexists with recordings_bp.index, resolved in B-1.3)
|
||||||
# - billing_bp at /checkout/* (routes added in B-2.7 and B-2.8)
|
# - billing_bp at /checkout/* (routes added in B-2.7 and B-2.8)
|
||||||
# - legal_bp at /legal/* (routes added in B-2.9)
|
# - legal_bp at /legal/* (routes added in B-2.9)
|
||||||
|
# NOTE: marketing_bp.landing at "/" is shadowed by recordings.index (registered
|
||||||
|
# earlier above). recordings.index dispatches anonymous users to landing() directly.
|
||||||
app.register_blueprint(marketing_bp)
|
app.register_blueprint(marketing_bp)
|
||||||
app.register_blueprint(billing_bp)
|
app.register_blueprint(billing_bp)
|
||||||
app.register_blueprint(legal_bp)
|
app.register_blueprint(legal_bp)
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ Allow: /tarifs
|
|||||||
Allow: /fonctionnalites
|
Allow: /fonctionnalites
|
||||||
Allow: /conformite
|
Allow: /conformite
|
||||||
Allow: /contact
|
Allow: /contact
|
||||||
Allow: /blog
|
Allow: /blog/
|
||||||
Allow: /legal/
|
Allow: /legal/
|
||||||
Disallow: /api/
|
Disallow: /api/
|
||||||
Disallow: /admin
|
Disallow: /admin
|
||||||
@@ -26,23 +26,46 @@ Disallow: /oublie
|
|||||||
Disallow: /verifier-email
|
Disallow: /verifier-email
|
||||||
Disallow: /webhooks/
|
Disallow: /webhooks/
|
||||||
|
|
||||||
# Search/AI crawlers explicitly allowed on public marketing surface
|
# Google-Extended (Bard/Gemini training): explicit opt-in to public marketing
|
||||||
User-agent: Googlebot
|
User-agent: Google-Extended
|
||||||
Allow: /
|
Allow: /
|
||||||
|
Allow: /tarifs
|
||||||
|
Allow: /fonctionnalites
|
||||||
|
Allow: /conformite
|
||||||
|
Allow: /contact
|
||||||
|
Allow: /blog/
|
||||||
|
Allow: /legal/
|
||||||
|
Disallow: /api/
|
||||||
|
Disallow: /admin
|
||||||
|
Disallow: /account
|
||||||
|
Disallow: /share/
|
||||||
|
Disallow: /app/
|
||||||
|
Disallow: /checkout
|
||||||
|
Disallow: /login
|
||||||
|
Disallow: /signup
|
||||||
|
Disallow: /oublie
|
||||||
|
Disallow: /verifier-email
|
||||||
|
Disallow: /webhooks/
|
||||||
|
|
||||||
User-agent: Bingbot
|
# ChatGPT-User (on-demand browsing): explicit opt-in to public marketing
|
||||||
Allow: /
|
User-agent: ChatGPT-User
|
||||||
|
|
||||||
User-agent: ClaudeBot
|
|
||||||
Allow: /
|
|
||||||
|
|
||||||
User-agent: GPTBot
|
|
||||||
Allow: /
|
|
||||||
|
|
||||||
User-agent: PerplexityBot
|
|
||||||
Allow: /
|
|
||||||
|
|
||||||
User-agent: Applebot
|
|
||||||
Allow: /
|
Allow: /
|
||||||
|
Allow: /tarifs
|
||||||
|
Allow: /fonctionnalites
|
||||||
|
Allow: /conformite
|
||||||
|
Allow: /contact
|
||||||
|
Allow: /blog/
|
||||||
|
Allow: /legal/
|
||||||
|
Disallow: /api/
|
||||||
|
Disallow: /admin
|
||||||
|
Disallow: /account
|
||||||
|
Disallow: /share/
|
||||||
|
Disallow: /app/
|
||||||
|
Disallow: /checkout
|
||||||
|
Disallow: /login
|
||||||
|
Disallow: /signup
|
||||||
|
Disallow: /oublie
|
||||||
|
Disallow: /verifier-email
|
||||||
|
Disallow: /webhooks/
|
||||||
|
|
||||||
Sitemap: https://dictia.pages.dev/sitemap.xml
|
Sitemap: https://dictia.pages.dev/sitemap.xml
|
||||||
|
|||||||
Reference in New Issue
Block a user