How do I override robots.txt for a nopcommerce website?

Question

robots.txt is generated by a NopCommerce Controller.

I need to edit it or have a custom one.

How do I do that?

Tried just placing my plain robots.txt to the root of website - did't work.

Answer 1

You need to place a plain text file named "robots.custom.txt" to the root of your website (for 4.0 and higher it is not the wwwroot directory) and its contents will be shown as robots.txt.

Answer 2

You can override PrepareRobotsTextFile factory from your plugin and write your code there.

For override CommonModelFactory, First create class for over-ride - RobotFactory.cs

 public class RobotFactory : CommonModelFactory
{
    #region Fields

    private readonly ICategoryService _categoryService;
    private readonly IProductService _productService;
    private readonly IManufacturerService _manufacturerService;
    private readonly ITopicService _topicService;
    private readonly ILanguageService _languageService;
    private readonly ICurrencyService _currencyService;
    private readonly ILocalizationService _localizationService;
    private readonly IWorkContext _workContext;
    private readonly IStoreContext _storeContext;
    private readonly ISitemapGenerator _sitemapGenerator;
    private readonly IThemeContext _themeContext;
    private readonly IThemeProvider _themeProvider;
    private readonly IForumService _forumservice;
    private readonly IGenericAttributeService _genericAttributeService;
    private readonly IWebHelper _webHelper;
    private readonly IPermissionService _permissionService;
    private readonly IStaticCacheManager _cacheManager;
    private readonly IPageHeadBuilder _pageHeadBuilder;
    private readonly IPictureService _pictureService;
    private readonly IHostingEnvironment _hostingEnvironment;
    private readonly IProductTagService _productTagService;
    private readonly CatalogSettings _catalogSettings;
    private readonly StoreInformationSettings _storeInformationSettings;
    private readonly CommonSettings _commonSettings;
    private readonly BlogSettings _blogSettings;
    private readonly NewsSettings _newsSettings;
    private readonly ForumSettings _forumSettings;
    private readonly LocalizationSettings _localizationSettings;
    private readonly CaptchaSettings _captchaSettings;
    private readonly VendorSettings _vendorSettings;
    private readonly IUrlRecordService _urlRecordService;
    private readonly ISeoService _seoService;
    private readonly IStoreService _storeService;
    private readonly ISettingService _settingService;

    #endregion

    #region Ctor

    public RobotFactory(ICategoryService categoryService, IProductService productService, IManufacturerService manufacturerService,
        ITopicService topicService, ILanguageService languageService, ICurrencyService currencyService,
        ILocalizationService localizationService, IWorkContext workContext, IStoreContext storeContext,
        ISitemapGenerator sitemapGenerator, IThemeContext themeContext, IThemeProvider themeProvider, IForumService forumService,
        IGenericAttributeService genericAttributeService, IWebHelper webHelper, IPermissionService permissionService,
        IStaticCacheManager cacheManager, IPageHeadBuilder pageHeadBuilder, IPictureService pictureService,
        IHostingEnvironment hostingEnvironment, CatalogSettings catalogSettings, StoreInformationSettings storeInformationSettings,
        CommonSettings commonSettings, BlogSettings blogSettings, NewsSettings newsSettings, ForumSettings forumSettings,
        LocalizationSettings localizationSettings, CaptchaSettings captchaSettings, VendorSettings vendorSettings,
        IProductTagService productTagService, IUrlRecordService urlRecordService, ISeoService seoService,
        IStoreService storeService, ISettingService settingService)
        : base(categoryService, productService, manufacturerService, topicService, languageService, currencyService,
              localizationService, workContext, storeContext, sitemapGenerator, themeContext, themeProvider, forumService,
              genericAttributeService, webHelper, permissionService, cacheManager, pageHeadBuilder, pictureService,
              hostingEnvironment, catalogSettings, storeInformationSettings, commonSettings, blogSettings, newsSettings,
              forumSettings, localizationSettings, captchaSettings, vendorSettings, productTagService)
    {
        this._categoryService = categoryService;
        this._productService = productService;
        this._manufacturerService = manufacturerService;
        this._topicService = topicService;
        this._languageService = languageService;
        this._currencyService = currencyService;
        this._localizationService = localizationService;
        this._workContext = workContext;
        this._storeContext = storeContext;
        this._sitemapGenerator = sitemapGenerator;
        this._themeContext = themeContext;
        this._themeProvider = themeProvider;
        this._forumservice = forumService;
        this._genericAttributeService = genericAttributeService;
        this._webHelper = webHelper;
        this._permissionService = permissionService;
        this._cacheManager = cacheManager;
        this._pageHeadBuilder = pageHeadBuilder;
        this._pictureService = pictureService;
        this._hostingEnvironment = hostingEnvironment;
        this._catalogSettings = catalogSettings;
        this._storeInformationSettings = storeInformationSettings;
        this._commonSettings = commonSettings;
        this._blogSettings = blogSettings;
        this._newsSettings = newsSettings;
        this._forumSettings = forumSettings;
        this._localizationSettings = localizationSettings;
        this._captchaSettings = captchaSettings;
        this._vendorSettings = vendorSettings;
        this._productTagService = productTagService;
        this._urlRecordService = urlRecordService;
        this._seoService = seoService;
        this._storeService = storeService;
        this._settingService = settingService;
    }

    #endregion

    #region Methods

    /// <summary>
    /// Get robots.txt file
    /// </summary>
    /// <returns>Robots.txt file as string</returns>
    public override string PrepareRobotsTextFile()
    {
        var storeScope = _storeContext.CurrentStore.Id;
        var seoSettings = _settingService.LoadSetting<SEOPluginSettings>(storeScope);

        var sb = new StringBuilder();

        //if robots.custom.txt exists, let's use it instead of hard-coded data below
        var robotsFilePath = System.IO.Path.Combine(CommonHelper.MapPath("~/"), "robots.custom.txt");
        if (System.IO.File.Exists(robotsFilePath))
        {
            //the robots.txt file exists
            var robotsFileContent = System.IO.File.ReadAllText(robotsFilePath);
            sb.Append(robotsFileContent);
        }
        else
        {
            //doesn't exist. Let's generate it (default behavior)
            var disallowPaths = new List<string>
            {
                // write your code here
            };
            var localizableDisallowPaths = new List<string>
            {
                // write your code here
            };

            // write your code here
        }

        return sb.ToString();
    }

    #endregion
}

Now you've register service in DependencyRegistrar.cs file

public class DependencyRegistrar : IDependencyRegistrar, INopStartup
{
    private const string CONTEXT_NAME = "nop_object_context__view_robot";

    public void Register(ContainerBuilder builder, ITypeFinder typeFinder, NopConfig config)
    {

        builder.RegisterType<RobotFactory>().As<ICommonModelFactory>().InstancePerLifetimeScope();
    }

    public void ConfigureServices(IServiceCollection services, IConfigurationRoot configuration)
    {
        services.Configure<RazorViewEngineOptions>(options =>
        {
            options.ViewLocationExpanders.Add(new CustomViewLocationExpander());
        });
    }

    public void Configure(IApplicationBuilder application)
    {
    }

    public int Order
    {
        get { return 100; }
    }
}

This will overide robot.txt file contents.