Innova-AI/dictia-public
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

feat(marketing): exempt public blueprints from noindex + fix / route collision

Browse Source 
- add_no_crawl_headers now skips marketing.*, legal.*, billing.success,
  static, and robots_txt endpoints via _is_public_indexable_endpoint
  helper; all other routes keep the X-Robots-Tag noindex header
- recordings.index drops @login_required and instead redirects
  anonymous users to marketing.landing, resolving the URL-map
  collision between recordings_bp and marketing_bp at "/"
- robots.txt rewritten: public marketing pages and /legal/* allowed,
  /api/, /admin, /account, /share/, /app/, /checkout, /login, /signup,
  /webhooks/ disallowed; Googlebot, Bingbot, ClaudeBot, GPTBot,
  PerplexityBot, Applebot explicitly allowed
- New tests/test_no_crawl_headers.py (14 tests) covers exemption
  helper + integration on /, /robots.txt, /static, /admin, /login
- New tests/test_marketing_root_redirect.py (4 tests) verifies
  anonymous users at / never get a /login redirect

Tests verified via AST + logic walkthrough; pytest blocked on Windows
by pre-existing fcntl import in src/init_db.py (B-1.2 limitation).
This commit is contained in:
Allison
2026-04-27 16:28:55 -04:00
parent 55ae09431d
commit 1071e56173
5 changed files with 299 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/api/recordings.py
View File

@@ -1357,8 +1357,20 @@ def reset_status(recording_id):
@recordings_bp.route('/')
@login_required
def index():
"""Root route handler.
Anonymous users are redirected to the marketing landing page so the
public site is reachable at "/". Authenticated users continue to see
the recordings dashboard (legacy Speakr UI).
Phase 1 of marketing redesign 2026 (B-1.3) replaced the previous
@login_required decorator with this inline check to resolve the route
collision between recordings_bp.index and marketing_bp.landing.
"""
if not current_user.is_authenticated:
return redirect(url_for('marketing.landing'))
# Check if user is a group admin
is_team_admin = GroupMembership.query.filter_by(
user_id=current_user.id,

28
src/app.py
View File

@@ -654,12 +654,40 @@ from src.config.startup import initialize_file_monitor, get_file_monitor_functio
run_startup_tasks(app)
# --- No-Crawl System: HTTP Headers ---
# Endpoints that must remain indexable by search engines and AI crawlers.
# Public marketing/legal/billing-success pages are exempted from the
# X-Robots-Tag noindex header so they can be discovered (Loi 25 transparency,
# GEO/SEO strategy). All other routes (api, admin, account, share, app, auth,
# recordings dashboard, etc.) keep the noindex header as defense-in-depth.
_PUBLIC_INDEXABLE_PREFIXES = ('marketing.', 'legal.')
_PUBLIC_INDEXABLE_ENDPOINTS = frozenset({
'billing.success', # post-payment confirmation page (added in B-2.7)
'robots_txt', # served from /robots.txt
'static', # static asset serving
})
def _is_public_indexable_endpoint(endpoint):
"""Return True if the resolved endpoint should NOT receive noindex headers."""
if not endpoint:
return False
if endpoint in _PUBLIC_INDEXABLE_ENDPOINTS:
return True
return endpoint.startswith(_PUBLIC_INDEXABLE_PREFIXES)
@app.after_request
def add_no_crawl_headers(response):
"""
Add HTTP headers to discourage search engine crawling and indexing.
This provides defense-in-depth alongside robots.txt and meta tags.
Marketing pages, legal pages, and the post-payment success page are
exempted so they remain indexable by search engines and AI crawlers.
"""
if _is_public_indexable_endpoint(request.endpoint):
return response
response.headers['X-Robots-Tag'] = 'noindex, nofollow, noarchive, nosnippet, noimageindex'
return response

89
static/robots.txt
View File

@@ -1,65 +1,48 @@
# DictIA - Block all web crawlers and search engines
# This application contains private user data and should not be indexed
# DictIA - robots.txt
# Updated 2026-04-27 for marketing redesign (Task B-1.3)
#
# Public marketing pages (root, /tarifs, /fonctionnalites, /conformite,
# /contact, /blog) and legal pages (/legal/*) are indexable.
# Application routes (/api, /admin, /account, /share, /app, /checkout,
# /login, /signup, /webhooks) remain blocked.
User-agent: *
Disallow: /
Allow: /
Allow: /tarifs
Allow: /fonctionnalites
Allow: /conformite
Allow: /contact
Allow: /blog
Allow: /legal/
Disallow: /api/
Disallow: /admin
Disallow: /account
Disallow: /share/
Disallow: /app/
Disallow: /checkout
Disallow: /login
Disallow: /signup
Disallow: /oublie
Disallow: /verifier-email
Disallow: /webhooks/
# Specific directives for major search engines
# Search/AI crawlers explicitly allowed on public marketing surface
User-agent: Googlebot
Disallow: /
User-agent: Googlebot-Image
Disallow: /
Allow: /
User-agent: Bingbot
Disallow: /
Allow: /
User-agent: Slurp
Disallow: /
User-agent: ClaudeBot
Allow: /
User-agent: DuckDuckBot
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: YandexBot
Disallow: /
User-agent: ia_archiver
Disallow: /
# AI Crawlers
User-agent: GPTBot
Disallow: /
Allow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: PerplexityBot
Allow: /
User-agent: CCBot
Disallow: /
User-agent: Applebot
Allow: /
User-agent: anthropic-ai
Disallow: /
User-agent: Claude-Web
Disallow: /
User-agent: cohere-ai
Disallow: /
# Social Media Crawlers
User-agent: facebookexternalhit
Disallow: /
User-agent: Twitterbot
Disallow: /
User-agent: LinkedInBot
Disallow: /
User-agent: Slackbot
Disallow: /
User-agent: Discordbot
Disallow: /
Sitemap: https://dictia.pages.dev/sitemap.xml

100
tests/test_marketing_root_redirect.py Normal file
View File

@@ -0,0 +1,100 @@
"""Verify the / route serves marketing.landing for anonymous users
instead of redirecting them to /login (Task B-1.3).
Before B-1.3, recordings_bp.index was decorated with @login_required and
redirected anonymous users to /login. After B-1.3, anonymous users land
on the public marketing site; only authenticated users see the legacy
Speakr dashboard.
Pattern: no conftest.py, env vars set at module load time, then import
src.app.app directly. Mirrors tests/test_blueprint_registration.py.
"""
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
os.environ.setdefault('SQLALCHEMY_DATABASE_URI', 'sqlite:///:memory:')
os.environ.setdefault('SECRET_KEY', 'test-secret-key-for-marketing-root-redirect')
from src.app import app # noqa: E402
def test_anonymous_user_at_root_does_not_go_to_login():
"""Anonymous user GET / must NOT be redirected to /login."""
client = app.test_client()
response = client.get('/', follow_redirects=False)
if response.status_code in (301, 302, 303, 307, 308):
location = response.headers.get('Location', '')
assert '/login' not in location, (
f"Expected anonymous user to see marketing, but redirected to: {location}"
)
def test_anonymous_user_at_root_sees_marketing_or_redirects_to_marketing():
"""Anonymous user GET / must see marketing landing (200) OR redirect to marketing.landing."""
client = app.test_client()
response = client.get('/', follow_redirects=False)
if response.status_code in (301, 302, 303, 307, 308):
# Acceptable: marketing landing is served via redirect from
# recordings_bp.index when the URL map prefers the recordings rule.
location = response.headers.get('Location', '')
assert '/login' not in location, (
f"Anonymous user redirected to login instead of marketing: {location}"
)
# The redirect target should be the marketing landing or root itself.
# Because marketing.landing is mounted at '/', url_for('marketing.landing')
# produces '/' — the redirect Location should not point back at any
# private route.
assert ('/admin' not in location and '/account' not in location), (
f"Unexpected redirect from / : {location}"
)
else:
assert response.status_code == 200, (
f"Expected 200 at / for anonymous, got {response.status_code}"
)
body_lower = response.data.lower()
assert (b'marketing' in body_lower) or (b'dictia' in body_lower), (
"Expected marketing landing content at / for anonymous user"
)
def test_root_route_resolves_to_recordings_index():
"""The recordings.index handler must still own '/' in the URL map.
The collision between recordings.index and marketing.landing is
deliberate: recordings.index is registered first and intercepts the
request, then redirects anonymous users to marketing.landing
(which itself maps to '/'). This test pins that contract so a
future refactor can't silently swap the registration order.
"""
rules_for_root = [
r for r in app.url_map.iter_rules() if str(r) == '/'
]
endpoints = {r.endpoint for r in rules_for_root}
# Both endpoints must exist for the redirect chain to work.
assert 'recordings.index' in endpoints, (
f"Expected recordings.index to own '/', got endpoints: {endpoints}"
)
assert 'marketing.landing' in endpoints, (
f"Expected marketing.landing also at '/', got endpoints: {endpoints}"
)
def test_index_handler_no_longer_login_required():
"""recordings.index must not have @login_required after B-1.3.
We assert this by hitting '/' anonymously and confirming the response
is NOT a Flask-Login redirect to /login. (Flask-Login emits a
302 with a `next` query parameter pointing back to '/'.)
"""
client = app.test_client()
response = client.get('/', follow_redirects=False)
if response.status_code in (301, 302, 303, 307, 308):
location = response.headers.get('Location', '')
assert 'next=' not in location, (
f"Detected Flask-Login redirect at /: {location} -- "
"@login_required appears to still be on recordings.index"
)

122
tests/test_no_crawl_headers.py Normal file
View File

@@ -0,0 +1,122 @@
"""Verify the no-crawl X-Robots-Tag is exempted for marketing/legal/billing
blueprints (Task B-1.3).
Marketing pages must be indexable by search engines and AI crawlers
(Loi 25 transparency, GEO/SEO strategy). The after_request hook in
src/app.py keeps emitting `X-Robots-Tag: noindex, nofollow, ...` for
private routes (/api/*, /admin, /account, /share, /app, /auth/*,
recordings dashboard).
Pattern: no conftest.py, env vars set at module load time, then import
src.app.app directly. Mirrors tests/test_blueprint_registration.py.
"""
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
os.environ.setdefault('SQLALCHEMY_DATABASE_URI', 'sqlite:///:memory:')
os.environ.setdefault('SECRET_KEY', 'test-secret-key-for-no-crawl-headers')
from src.app import app, _is_public_indexable_endpoint # noqa: E402
def test_marketing_root_has_no_noindex_header():
"""Marketing root '/' must NOT have X-Robots-Tag noindex header."""
client = app.test_client()
response = client.get('/')
# Anonymous user gets the marketing landing (B-1.3 swap of recordings.index).
robots_tag = response.headers.get('X-Robots-Tag', '')
assert 'noindex' not in robots_tag.lower(), (
f"Expected no noindex on /, got X-Robots-Tag: {robots_tag!r}"
)
def test_robots_txt_route_has_no_noindex_header():
"""The /robots.txt response itself must not be noindexed (it's a directive)."""
client = app.test_client()
response = client.get('/robots.txt')
robots_tag = response.headers.get('X-Robots-Tag', '')
assert 'noindex' not in robots_tag.lower(), (
f"Expected no noindex on /robots.txt, got X-Robots-Tag: {robots_tag!r}"
)
def test_static_assets_have_no_noindex_header():
"""Static assets must not carry X-Robots-Tag noindex."""
client = app.test_client()
# We don't need a real static file — the endpoint resolution is what
# the after_request hook keys off. Use a known-missing path; Flask's
# static handler still resolves request.endpoint to 'static'.
response = client.get('/static/this-does-not-exist.css')
robots_tag = response.headers.get('X-Robots-Tag', '')
assert 'noindex' not in robots_tag.lower(), (
f"Expected no noindex on /static/*, got X-Robots-Tag: {robots_tag!r}"
)
def test_private_route_still_has_noindex():
"""A private app route must STILL have noindex headers."""
client = app.test_client()
# /admin requires login; anonymous gets a redirect, but the after_request
# hook still runs on the redirect response. That redirect response must
# carry the noindex header (defense-in-depth).
response = client.get('/admin', follow_redirects=False)
robots_tag = response.headers.get('X-Robots-Tag', '')
assert 'noindex' in robots_tag.lower(), (
f"Expected noindex on /admin, got X-Robots-Tag: {robots_tag!r}"
)
def test_login_route_still_has_noindex():
"""The /login page is a private utility surface and must keep noindex."""
client = app.test_client()
response = client.get('/login', follow_redirects=False)
robots_tag = response.headers.get('X-Robots-Tag', '')
assert 'noindex' in robots_tag.lower(), (
f"Expected noindex on /login, got X-Robots-Tag: {robots_tag!r}"
)
# --- Direct unit tests on the helper predicate ---
def test_helper_marketing_endpoint_is_indexable():
assert _is_public_indexable_endpoint('marketing.landing') is True
def test_helper_legal_endpoint_is_indexable():
assert _is_public_indexable_endpoint('legal.confidentialite') is True
def test_helper_billing_success_is_indexable():
assert _is_public_indexable_endpoint('billing.success') is True
def test_helper_billing_other_endpoints_not_indexable():
# Only billing.success is exempt; the rest of the checkout flow stays private.
assert _is_public_indexable_endpoint('billing.checkout') is False
assert _is_public_indexable_endpoint('billing.webhook') is False
def test_helper_static_endpoint_is_indexable():
assert _is_public_indexable_endpoint('static') is True
def test_helper_robots_endpoint_is_indexable():
assert _is_public_indexable_endpoint('robots_txt') is True
def test_helper_api_endpoint_not_indexable():
assert _is_public_indexable_endpoint('api.something') is False
def test_helper_recordings_index_not_indexable():
"""The dashboard at '/' (authenticated branch) must keep noindex."""
assert _is_public_indexable_endpoint('recordings.index') is False
def test_helper_none_endpoint_not_indexable():
assert _is_public_indexable_endpoint(None) is False
assert _is_public_indexable_endpoint('') is False