import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
public static void main(String args[]) throws Exception {
    HttpProtocol protocol = new HttpProtocol();

    String url = args[0];
    Config conf = ConfUtils.loadConf(args[1]);

    protocol.configure(conf);

    if (!protocol.skipRobots) {
        BaseRobotRules rules = protocol.getRobotRules(url);
        System.out.println("is allowed : " + rules.isAllowed(url));
    }

    Metadata md = new Metadata();
    ProtocolResponse response = protocol.getProtocolOutput(url, md);
    System.out.println(url);
    System.out.println(response.getMetadata());
    System.out.println(response.getStatusCode());
    System.out.println(response.getContent().length);
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
@Override
public String filter(URL sourceUrl, Metadata sourceMetadata,
        String urlToFilter) {
    URL target;
    try {
        target = new URL(urlToFilter);
    } catch (MalformedURLException e) {
        return null;
    }

    // check whether the source and target have the same hostname
    if (limitToSameHost) {
        if (!target.getHost().equalsIgnoreCase(sourceUrl.getHost())) {
            return urlToFilter;
        }
    }

    BaseRobotRules rules = robots.getRobotRulesSet(
            factory.getProtocol(target), urlToFilter);
    if (!rules.isAllowed(urlToFilter)) {
        return null;
    }
    return urlToFilter;
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
public static void parseRobotsTxt(String userAgent, String robotsUrl, String robotsTxt, HtmlAnalysisResult result) {
    result.setRobotsTxt(robotsTxt);
    SimpleRobotRulesParser robotsParser = new SimpleRobotRulesParser();
    BaseRobotRules robotRules = robotsParser.parseContent(robotsUrl, robotsTxt.getBytes(), null, userAgent);
    result.setRobotsAllowedAll(robotRules.isAllowAll());
    result.setRobotsAllowedNone(robotRules.isAllowNone());
    result.setRobotsAllowedHome(robotRules.isAllowed("/"));
    result.setRobotsSitemaps(robotRules.getSitemaps());
    result.setRobotsCrawlDelay(robotRules.getCrawlDelay());
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
protected BaseRobotRules getRules(URI uri) {
    try {
        return RobotUtils.getRobotRules(fetcher, parser,
                new URL(uri.getScheme(), uri.getHost(), uri.getPort(), ROBOTS_FILE_NAME));
    } catch (MalformedURLException e) {
        LOGGER.error("URL of robots.txt file is malformed. Returning rules for HTTP 400.");
        return parser.failedFetch(400);
    }
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
@Override
public long getMinWaitingTime(URI uri) {
    BaseRobotRules rules = getRules(uri);
    long delay = rules.getCrawlDelay();
    if (delay <= 0) {
        return 0;
    } else {
        return delay;
    }
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
public BaseRobotRules getRobotRulesSet(Protocol protocol, Text url) {
  URL u = null;
  try {
    u = new URL(url.toString());
  } catch (Exception e) {
    return EMPTY_RULES;
  }
  return getRobotRulesSet(protocol, u);
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
public BaseRobotRules getRobotRulesSet(Protocol protocol, String url) {
    URL u = null;
    try {
        u = new URL(url);
    } catch (Exception e) {
        return EMPTY_RULES;
    }
    return getRobotRulesSet(protocol, u);
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
public boolean isDisallowedByRobots(LinkRelevance link) {
    String hostname = link.getURL().getHost();
    BaseRobotRules rules = robotRulesMap.get(hostname);
    return rules != null && !rules.isAllowed(link.getURL().toString());
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
private void processRobot(LinkRelevance link, FetchedResult response, boolean fetchFailed) {
    
    BaseRobotRules robotRules;
    if(fetchFailed || response == null) {
        robotRules = parser.failedFetch(HttpStatus.SC_GONE);
    }
    else {
        String contentType = response.getContentType();
        boolean isPlainText = (contentType != null) && (contentType.startsWith("text/plain"));
        if ((response.getNumRedirects() > 0) && !isPlainText) {
            robotRules = parser.failedFetch(HttpStatus.SC_GONE);
        } else {
            robotRules = parser.parseContent(
                response.getFetchedUrl(),
                response.getContent(),
                response.getContentType(),
                userAgentName   
            );
        }
    }
    
    try {
        RobotsData robotsData = new RobotsData(link, robotRules);
        linkStorage.insert(robotsData);
    } catch (Exception e) {
        logger.error("Failed to insert robots.txt data into link storage.", e);
    }
    
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
@Override
public BaseRobotRules failedFetch(int httpStatusCode) {
	ExtendedRobotRules result;

	if ((httpStatusCode >= 200) && (httpStatusCode < 300)) {
		throw new IllegalStateException("Can't use status code constructor with 2xx response");
	} else if ((httpStatusCode >= 300) && (httpStatusCode < 400)) {
		// Should only happen if we're getting endless redirects (more than
		// our follow limit), so
		// treat it as a temporary failure.
		result = new ExtendedRobotRules(RobotRulesMode.ALLOW_NONE);
		result.setDeferVisits(true);
	} else if ((httpStatusCode >= 400) && (httpStatusCode < 500)) {
		// Some sites return 410 (gone) instead of 404 (not found), so treat
		// as the same.
		// Actually treat all (including forbidden) as "no robots.txt", as
		// that's what Google
		// and other search engines do.
		result = new ExtendedRobotRules(RobotRulesMode.ALLOW_ALL);
	} else {
		// Treat all other status codes as a temporary failure.
		result = new ExtendedRobotRules(RobotRulesMode.ALLOW_NONE);
		result.setDeferVisits(true);
	}

	return result;
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
public BaseRobotRules getRobotRulesSet(Protocol protocol, String url) {
    URL u;
    try {
        u = new URL(url);
    } catch (Exception e) {
        return EMPTY_RULES;
    }
    return getRobotRulesSet(protocol, u);
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
@Override
public boolean isUriCrawlable(URI uri) {
    BaseRobotRules rules = getRules(uri);
    return rules.isAllowed(uri.toString());
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
/**
 * Get the robots rules for a given url
 */
public BaseRobotRules getRobotRules(Text url, CrawlDatum datum) {
  return robots.getRobotRulesSet(this, url);
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
/**
 * The hosts for which the caching of robots rules is yet to be done, it sends
 * a Ftp request to the host corresponding to the {@link URL} passed, gets
 * robots file, parses the rules and caches the rules object to avoid re-work
 * in future.
 * 
 * @param ftp
 *          The {@link Protocol} object
 * @param url
 *          URL
 * 
 * @return robotRules A {@link BaseRobotRules} object for the rules
 */
public BaseRobotRules getRobotRulesSet(Protocol ftp, URL url) {

  String protocol = url.getProtocol().toLowerCase(); // normalize to lower
                                                     // case
  String host = url.getHost().toLowerCase(); // normalize to lower case

  if (LOG.isTraceEnabled() && isWhiteListed(url)) {
    LOG.trace("Ignoring robots.txt (host is whitelisted) for URL: {}", url);
  }

  BaseRobotRules robotRules = CACHE.get(protocol + ":" + host);

  if (robotRules != null) {
    return robotRules; // cached rule
  } else if (LOG.isTraceEnabled()) {
    LOG.trace("cache miss " + url);
  }

  boolean cacheRule = true;

  if (isWhiteListed(url)) {
    // check in advance whether a host is whitelisted
    // (we do not need to fetch robots.txt)
    robotRules = EMPTY_RULES;
    LOG.info("Whitelisted host found for: {}", url);
    LOG.info("Ignoring robots.txt for all URLs from whitelisted host: {}", host);

  } else {
    try {
      Text robotsUrl = new Text(new URL(url, "/robots.txt").toString());
      ProtocolOutput output = ((Ftp) ftp).getProtocolOutput(robotsUrl,
          new CrawlDatum());
      ProtocolStatus status = output.getStatus();

      if (status.getCode() == ProtocolStatus.SUCCESS) {
        robotRules = parseRules(url.toString(), output.getContent()
            .getContent(), CONTENT_TYPE, agentNames);
      } else {
        robotRules = EMPTY_RULES; // use default rules
      }
    } catch (Throwable t) {
      if (LOG.isInfoEnabled()) {
        LOG.info("Couldn't get robots.txt for " + url + ": " + t.toString());
      }
      cacheRule = false; // try again later to fetch robots.txt
      robotRules = EMPTY_RULES;
    }

  }

  if (cacheRule)
    CACHE.put(protocol + ":" + host, robotRules); // cache rules for host

  return robotRules;
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
public BaseRobotRules getRobotRules(Text url, CrawlDatum datum) {
  return robots.getRobotRulesSet(this, url);
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
@Override
public BaseRobotRules getRobotRules(String url) {
    if (this.skipRobots)
        return RobotRulesParser.EMPTY_RULES;
    return robots.getRobotRulesSet(this, url);
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
/**
 * Get the rules from robots.txt which applies for the given {@code url}.
 * Robot rules are cached for a unique combination of host, protocol, and
 * port. If no rules are found in the cache, a HTTP request is send to fetch
 * {{protocol://host:port/robots.txt}}. The robots.txt is then parsed and
 * the rules are cached to avoid re-fetching and re-parsing it again.
 * 
 * @param http
 *            The {@link Protocol} object
 * @param url
 *            URL robots.txt applies to
 * 
 * @return {@link BaseRobotRules} holding the rules from robots.txt
 */
@Override
public BaseRobotRules getRobotRulesSet(Protocol http, URL url) {

    String cacheKey = getCacheKey(url);
    BaseRobotRules robotRules = CACHE.get(cacheKey);

    boolean cacheRule = true;

    if (robotRules == null) { // cache miss
        URL redir = null;
        LOG.trace("cache miss {}", url);
        try {
            ProtocolResponse response = http.getProtocolOutput(new URL(url,
                    "/robots.txt").toString(), Metadata.empty);

            // try one level of redirection ?
            if (response.getStatusCode() == 301
                    || response.getStatusCode() == 302) {
                String redirection = response.getMetadata().getFirstValue(
                        HttpHeaders.LOCATION);
                if (StringUtils.isNotBlank(redirection)) {
                    if (!redirection.startsWith("http")) {
                        // RFC says it should be absolute, but apparently it
                        // isn't
                        redir = new URL(url, redirection);
                    } else {
                        redir = new URL(redirection);
                    }
                    response = http.getProtocolOutput(redir.toString(),
                            Metadata.empty);
                }
            }

            if (response.getStatusCode() == 200) // found rules: parse them
            {
                String ct = response.getMetadata().getFirstValue(
                        HttpHeaders.CONTENT_TYPE);
                robotRules = parseRules(url.toString(),
                        response.getContent(), ct, agentNames);
            } else if ((response.getStatusCode() == 403)
                    && (!allowForbidden))
                robotRules = FORBID_ALL_RULES; // use forbid all
            else if (response.getStatusCode() >= 500) {
                cacheRule = false;
                robotRules = EMPTY_RULES;
            } else
                robotRules = EMPTY_RULES; // use default rules
        } catch (Throwable t) {
            LOG.info("Couldn't get robots.txt for {} : {}", url,
                    t.toString());
            cacheRule = false;
            robotRules = EMPTY_RULES;
        }

        if (cacheRule) {
            CACHE.put(cacheKey, robotRules); // cache rules for host
            if (redir != null
                    && !redir.getHost().equalsIgnoreCase(url.getHost())) {
                // cache also for the redirected host
                CACHE.put(getCacheKey(redir), robotRules);
            }
        }
    }
    return robotRules;
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
public Frontier(String directory, int maxCacheUrlsSize, DB persistentHashtableBackend) {
    this.urlRelevance = new PersistentHashtable<>(directory, maxCacheUrlsSize,
            LinkRelevance.class, persistentHashtableBackend);
    this.robotRulesMap = new PersistentHashtable<>(directory + "_robots", maxCacheUrlsSize,
            BaseRobotRules.class, persistentHashtableBackend);
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
public RobotsData(LinkRelevance link, BaseRobotRules robotRules) {
    this.link = link;
    this.robotRules = robotRules;
}
 

import crawlercommons.robots.BaseRobotRules; //导入依赖的package包/类
@Override
public BaseRobotRules getRobotRules(String url) {
    return RobotRulesParser.EMPTY_RULES;
}