# Robots.txt for Nivk.com
# Optimized for Google, Bing, AI crawlers (ChatGPT, Claude, Perplexity, etc.), and all search engines
# Last updated: 2026-01-29
# 
# IMPORTANT: This site welcomes ALL crawlers and AI systems
# - No pages are blocked
# - All content is accessible
# - Sitemaps provided for easy discovery
# - AI-specific files available: /ai.txt, /llms.txt, /llm.txt, /llms-full.txt

# ============================================
# GOOGLE CRAWLERS
# ============================================

# Googlebot (Main Google crawler)
User-agent: Googlebot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# Googlebot-Image (Image crawler)
User-agent: Googlebot-Image
Allow: /
Sitemap: https://nivk.com/sitemap-index.xml

# Googlebot-Video (Video crawler)
User-agent: Googlebot-Video
Allow: /
Sitemap: https://nivk.com/sitemap-index.xml

# Google-Extended (Google AI training crawler for Gemini, SGE)
User-agent: Google-Extended
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# ============================================
# MICROSOFT / BING CRAWLERS
# ============================================

# Bingbot (Main Bing crawler)
User-agent: Bingbot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# BingPreview (Bing preview bot)
User-agent: BingPreview
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# msnbot (Microsoft MSN bot)
User-agent: msnbot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# ============================================
# OTHER SEARCH ENGINES
# ============================================

# Slurp (Yahoo)
User-agent: Slurp
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# DuckDuckBot (DuckDuckGo)
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# Baiduspider (Baidu - China)
User-agent: Baiduspider
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# Yandex (Russian search engine)
User-agent: Yandex
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# Applebot (Apple Search)
User-agent: Applebot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# Sogou (Chinese search engine)
User-agent: Sogou
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# PetalBot (Huawei Search)
User-agent: PetalBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# ============================================
# AI CRAWLERS & LLM TRAINING BOTS
# ============================================

# ChatGPT-User (OpenAI crawler for ChatGPT)
User-agent: ChatGPT-User
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# GPTBot (OpenAI crawler for training)
User-agent: GPTBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# OAI-SearchBot (OpenAI Search - New ChatGPT search)
User-agent: OAI-SearchBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# anthropic-ai (Anthropic Claude crawler)
User-agent: anthropic-ai
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# Claude-Web (Anthropic Claude web crawler)
User-agent: Claude-Web
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# ClaudeBot (Alternative Anthropic user agent)
User-agent: ClaudeBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# PerplexityBot (Perplexity AI search)
User-agent: PerplexityBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# Perplexity (Alternative)
User-agent: Perplexity
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# YouBot (You.com AI search)
User-agent: YouBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# CCBot (Common Crawl - used by many AI systems)
User-agent: CCBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# Diffbot (AI-powered web crawler)
User-agent: Diffbot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# cohere-ai (Cohere AI)
User-agent: cohere-ai
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# AI2Bot (Allen Institute for AI)
User-agent: AI2Bot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# Amazonbot (Amazon Alexa & Product Search AI)
User-agent: Amazonbot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# Bytespider (ByteDance/TikTok AI)
User-agent: Bytespider
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# DuckAssistBot (DuckDuckGo AI Assistant)
User-agent: DuckAssistBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# PhindBot (AI Code Search)
User-agent: PhindBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# KomoBot (Komo AI Search)
User-agent: KomoBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# iaskBot (iAsk AI Search)
User-agent: iaskBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# WritesonicBot (Writesonic AI Content)
User-agent: WritesonicBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# JasperBot (Jasper AI Content)
User-agent: JasperBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# Meta-ExternalAgent (Meta/Facebook AI)
User-agent: Meta-ExternalAgent
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# Meta-ExternalFetcher (Meta/Facebook AI Fetcher)
User-agent: Meta-ExternalFetcher
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# ImagesiftBot (Image AI Crawler)
User-agent: ImagesiftBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# Omgilibot (Webz.io AI Data)
User-agent: Omgilibot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# webzio-extended (Webz.io Extended)
User-agent: webzio-extended
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# ============================================
# SOCIAL MEDIA CRAWLERS
# ============================================

# Facebook External Hit (Link previews)
User-agent: facebookexternalhit
Allow: /
Crawl-delay: 0

# FacebookBot (Alternative)
User-agent: FacebookBot
Allow: /
Crawl-delay: 0

# Twitterbot (Twitter/X link previews)
User-agent: Twitterbot
Allow: /
Crawl-delay: 0

# LinkedInBot (LinkedIn link previews)
User-agent: LinkedInBot
Allow: /
Crawl-delay: 0

# WhatsApp (WhatsApp link previews)
User-agent: WhatsApp
Allow: /
Crawl-delay: 0

# TelegramBot (Telegram link previews)
User-agent: TelegramBot
Allow: /
Crawl-delay: 0

# Discordbot (Discord link previews)
User-agent: Discordbot
Allow: /
Crawl-delay: 0

# Slackbot (Slack link previews)
User-agent: Slackbot
Allow: /
Crawl-delay: 0

# ============================================
# SEO & ANALYTICS TOOLS
# ============================================

# SemrushBot (SEO analysis tool)
User-agent: SemrushBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# AhrefsBot (SEO analysis tool)
User-agent: AhrefsBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# MJ12bot (Majestic SEO crawler)
User-agent: MJ12bot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# DotBot (Moz crawler)
User-agent: DotBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# DataForSeoBot (DataForSEO crawler)
User-agent: DataForSeoBot
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# ============================================
# DEFAULT RULES
# ============================================

# Default rule for all other bots - ALLOW EVERYTHING
User-agent: *
Allow: /
Crawl-delay: 0
Sitemap: https://nivk.com/sitemap-index.xml

# ============================================
# HOST DIRECTIVE
# ============================================

# Canonical host (preferred domain)
Host: https://nivk.com

# ============================================
# SITEMAP LOCATION
# ============================================

# Primary sitemap index
Sitemap: https://nivk.com/sitemap-index.xml

# ============================================
# AI/LLM STRUCTURED DATA FILES
# ============================================

# llms.txt - Comprehensive AI-readable site information
# llm.txt - Condensed quick reference for LLMs
# llms-full.txt - Exhaustive detailed content for AI assistants
# ai.txt - AI crawler guidance and instructions
# These files help AI systems understand Nivk's offerings

LLMs-Info: https://nivk.com/llms.txt
LLMs-Full: https://nivk.com/llms-full.txt
LLM-Info: https://nivk.com/llm.txt
AI-Info: https://nivk.com/ai.txt

# ============================================
# FEEDS & DISCOVERY FILES
# ============================================

# RSS/Atom/JSON Feeds for blog content
RSS-Feed: https://nivk.com/rss.xml
Atom-Feed: https://nivk.com/atom.xml
JSON-Feed: https://nivk.com/feed.json

# OpenSearch for browser integration
OpenSearch: https://nivk.com/opensearch.xml

# Security information (RFC 9116)
Security: https://nivk.com/.well-known/security.txt

# Team credits
Humans: https://nivk.com/humans.txt

# Additional sitemaps
News-Sitemap: https://nivk.com/news-sitemap.xml
Image-Sitemap: https://nivk.com/image-sitemap.xml
Video-Sitemap: https://nivk.com/video-sitemap.xml

# HTML Site Index (for AI crawlers and users)
Site-Index: https://nivk.com/site-index.html