import net.htmlparser.jericho.HTMLElementName; //导入依赖的package包/类
private static boolean isValidTRTag(Tag tag) {
	Element parentElement=tag.getElement().getParentElement();
	if (parentElement==null) return false; // ignore TR elements without a parent
	if (parentElement.getStartTag().getUserData()!=VALID_MARKER) return false; // ignore TR elements who's parent is not valid
	if(parentElement.getName()==HTMLElementName.TABLE){
		return true;
	}
	if(parentElement.getName()==HTMLElementName.TBODY
			||parentElement.getName()==HTMLElementName.THEAD){
		Element gradParent=parentElement.getParentElement();
		if(gradParent==null){
			return false;
		}
		if (gradParent.getStartTag().getUserData()!=VALID_MARKER) return false;
		return (gradParent.getName()==HTMLElementName.TABLE);
	}else{
		return false;
	}
}
 

import net.htmlparser.jericho.HTMLElementName; //导入依赖的package包/类
public void printHTMLPage(Source source)
        throws UnsupportedEncodingException, IOException {
    List<StartTag> list = source.getAllStartTags(HTMLElementName.STYLE);
    Iterator<StartTag> iterator = list.iterator();
    String text = "";
    while (iterator.hasNext()) {
        final StartTag tag = iterator.next();
        final Segment s = new Segment(source, tag.getEnd(), tag
                .getElement().getEndTag().getBegin());
        text += s.toString();
    }
    Vector<ReplaceRight> rights = Style.getStyles(text, styles, counter);

    // генерація сторінки з оновленими стилями.

    list = source.getAllStartTags();
    iterator = list.iterator();
    StartTag startTag = null;
    while (iterator.hasNext()) {
        final StartTag st = iterator.next();
        if (HTMLElementName.BODY.equals(st.getName())) {
            startTag = st;
            break;
        }
    }
    if (startTag == null)
        return;

    final StartTag body = startTag;
    final OutputDocument document = new OutputDocument(source);
    while (iterator.hasNext()) {
        startTag = iterator.next();
        replaceAttrs(startTag, document, rights);
    }

    OutputStreamWriter writer = new OutputStreamWriter(this.out, "UTF-8");
    document.writeTo(writer, body.getEnd(), body.getElement().getEndTag()
            .getBegin());
    writer.flush();
}
 

import net.htmlparser.jericho.HTMLElementName; //导入依赖的package包/类
public static void main(final String[] args) {
    try {
        final FileInputStream is = new FileInputStream("d:/test.html");
        final FileOutputStream o = new FileOutputStream("d:/res1.txt");
        final Source source = new Source(is);
        final List<StartTag> list = source
                .getAllStartTags(HTMLElementName.STYLE);
        final Iterator<StartTag> iterator = list.iterator();
        String text = "";
        final PrintStream out = new PrintStream(o);
        while (iterator.hasNext()) {
            final StartTag tag = iterator.next();
            final Segment s = new Segment(source, tag.getEnd(), tag
                    .getElement().getEndTag().getBegin());
            text += s.toString();
            // out.println(text);
            // out.println("---------------");
        }
        is.close();
        final Vector<Style> styles = new Vector<Style>();
        getStyles(text, styles, createCounter());
        for (int i = 0; i < styles.size(); i++) {
            out.println(styles.get(i));
        }
        o.close();
    } catch (final IOException e) {
        e.printStackTrace();
    }
}
 

import net.htmlparser.jericho.HTMLElementName; //导入依赖的package包/类
/**
 * Updates relative source attributes to externally accessible abs paths
 * @param html the document
 * @param baseDir html location
 * @param out processed document
 */
protected void modifyDocument( Source html, PathOrigin baseDir, OutputDocument out ) {
  replaceUrlAttribute( html.getAllStartTags( HTMLElementName.LINK ), "href", baseDir, out );
  replaceUrlAttribute( html.getAllStartTags( HTMLElementName.SCRIPT ), "src", baseDir, out );
  replaceUrlAttribute( html.getAllStartTags( HTMLElementName.IMG ), "src", baseDir, out );
  //int insertPos = html.getFirstElement( HTMLElementName.HEAD ).getEndTag().getBegin();
  //out.insert( insertPos, getCodeSnippet( getBackendAssignments( getUrlProvider() ) ) );
}
 

import net.htmlparser.jericho.HTMLElementName; //导入依赖的package包/类
private static boolean processTag(Tag tag, OutputDocument outputDocument) {
	String elementName=tag.getName();
	if (!VALID_ELEMENT_NAMES.contains(elementName)){
		//System.out.println("Not ok tag:!"+elementName+"!");
		return false;
	}
	if (tag.getTagType()==StartTagType.NORMAL) {
		Element element=tag.getElement();
		if (elementName==HTMLElementName.THEAD && !isValidTbodyTHeadTag(tag)) return false;
		if (elementName==HTMLElementName.TBODY && !isValidTbodyTHeadTag(tag)) return false;
		if (elementName==HTMLElementName.TR && !isValidTRTag(tag)) return false;
		if (elementName==HTMLElementName.TD && !isValidTDTHTag(tag)) return false;
		if (elementName==HTMLElementName.TH && !isValidTDTHTag(tag)) return false;
		if (HTMLElements.getEndTagRequiredElementNames().contains(elementName)) {
			if (element.getEndTag()==null) return false; // reject start tag if its required end tag is missing
		} else if (HTMLElements.getEndTagOptionalElementNames().contains(elementName)) {
			if (elementName==HTMLElementName.LI && !isValidLITag(tag)) return false; // reject invalid LI tags
			if (element.getEndTag()==null) outputDocument.insert(element.getEnd(),getEndTagHTML(elementName)); // insert optional end tag if it is missing
		}
		outputDocument.replace(tag,getStartTagHTML(element.getStartTag()));
	} else if (tag.getTagType()==EndTagType.NORMAL) {
		if (tag.getElement()==null) return false; // reject end tags that aren't associated with a start tag
		if (elementName==HTMLElementName.THEAD && !isValidTbodyTHeadTag(tag)) return false;
		if (elementName==HTMLElementName.TBODY && !isValidTbodyTHeadTag(tag)) return false;
		if (elementName==HTMLElementName.TR && !isValidTRTag(tag)) return false;
		if (elementName==HTMLElementName.TD && !isValidTDTHTag(tag)) return false;
		if (elementName==HTMLElementName.TH && !isValidTDTHTag(tag)) return false;
		if (elementName==HTMLElementName.LI && !isValidLITag(tag)) return false; // reject invalid LI tags
		outputDocument.replace(tag,getEndTagHTML(elementName));
	} else {
		return false; // reject abnormal tags
	}
	return true;
}
 

import net.htmlparser.jericho.HTMLElementName; //导入依赖的package包/类
private static String getMetaValue(Source source, String key) {
	for (int pos=0; pos<source.length();) {
		StartTag startTag=source.getNextStartTag(pos,"name",key,false);
		if (startTag==null) return null;
		if (startTag.getName()==HTMLElementName.META)
			return startTag.getAttributeValue("content"); // Attribute values are automatically decoded
		pos=startTag.getEnd();
	}
	return null;
}
 

import net.htmlparser.jericho.HTMLElementName; //导入依赖的package包/类
private boolean processTag(Tag tag, OutputDocument output) {
	String elementName = tag.getName().toLowerCase();
	if (!allowedTags.contains(elementName))
		return false;
	if (tag.getTagType() == StartTagType.NORMAL) {
		Element element = tag.getElement();
		if (HTMLElements.getEndTagRequiredElementNames().contains(
				elementName)) {
			if (element.getEndTag() == null)
				return false; // reject start tag if its required end tag is
								// missing
		} else if (HTMLElements.getEndTagOptionalElementNames().contains(
				elementName)) {
			if (elementName == HTMLElementName.LI && !isValidLITag(tag))
				return false; // reject invalid LI tags
			if (element.getEndTag() == null)
				output.insert(element.getEnd(), getEndTagHTML(elementName)); // insert
																				// optional
																				// end
																				// tag
																				// if
																				// it
																				// is
																				// missing
		}
		output.replace(tag, getStartTagHTML(element.getStartTag()));
	} else if (tag.getTagType() == EndTagType.NORMAL) {
		if (tag.getElement() == null)
			return false; // reject end tags that aren't associated with a
							// start tag
		if (elementName == HTMLElementName.LI && !isValidLITag(tag))
			return false; // reject invalid LI tags
		output.replace(tag, getEndTagHTML(elementName));
	} else {
		return false; // reject abnormal tags
	}
	return true;
}
 

import net.htmlparser.jericho.HTMLElementName; //导入依赖的package包/类
private boolean isValidLITag(Tag tag) {
	Element parentElement = tag.getElement().getParentElement();
	if (parentElement == null)
		return false; // ignore LI elements without a parent
	if (parentElement.getStartTag().getUserData() != VALID_MARKER)
		return false; // ignore LI elements who's parent is not valid
	return parentElement.getName() == HTMLElementName.UL
			|| parentElement.getName() == HTMLElementName.OL; // only accept
																// LI tags
																// who's
																// immediate
																// parent is
																// UL or OL.
}
 

import net.htmlparser.jericho.HTMLElementName; //导入依赖的package包/类
private boolean processTag(Tag tag, OutputDocument output) {
	String elementName = tag.getName().toLowerCase();
	if (!allowedTags.contains(elementName))
		return false;
	if (tag.getTagType() == StartTagType.NORMAL) {
		Element element = tag.getElement();
		if (HTMLElements.getEndTagRequiredElementNames().contains(elementName)) {
			if (element.getEndTag() == null)
				return false; // reject start tag if its required end tag is
				              // missing
		} else if (HTMLElements.getEndTagOptionalElementNames().contains(elementName)) {
			if (elementName == HTMLElementName.LI && !isValidLITag(tag))
				return false; // reject invalid LI tags
			if (element.getEndTag() == null)
				// insert optional end tag if it is missing
				output.insert(element.getEnd(), getEndTagHTML(elementName));
		}
		output.replace(tag, getStartTagHTML(element.getStartTag()));
	} else if (tag.getTagType() == EndTagType.NORMAL) {
		if (tag.getElement() == null)
			return false; // reject end tags that aren't associated with a
			              // start tag
		if (elementName == HTMLElementName.LI && !isValidLITag(tag))
			return false; // reject invalid LI tags
		output.replace(tag, getEndTagHTML(elementName));
	} else {
		return false; // reject abnormal tags
	}
	return true;
}
 

import net.htmlparser.jericho.HTMLElementName; //导入依赖的package包/类
private boolean isValidLITag(Tag tag) {
	Element parentElement = tag.getElement().getParentElement();
	if (parentElement == null)
		return false; // ignore LI elements without a parent
	if (parentElement.getStartTag().getUserData() != VALID_MARKER)
		return false; // ignore LI elements who's parent is not valid
	// only accept LI tags who's immediate parent is UL or OL.
	return parentElement.getName() == HTMLElementName.UL || parentElement.getName() == HTMLElementName.OL; 
}
 

import net.htmlparser.jericho.HTMLElementName; //导入依赖的package包/类
public static ArrayList<Image> parseHtml(String html) {
	ArrayList<Image> images = new ArrayList<Image>();
	Source source = new Source(html);
	
	Element table = source.getAllElements(HTMLElementName.TABLE).get(0);
	Element div = table.getFirstElement().getFirstElement().getFirstElement().getFirstElement();
	
	ArrayList<Element> ps = new ArrayList<Element>(div.getAllElements(HTMLElementName.P));
	
	boolean isReallyP = true;
	for (Element p : ps) {
		if (p.getContent().toString().contains("이미지 결과 더 보기")) {
			isReallyP = false;
		}
		
		if (isReallyP && p.getAllElements(HTMLElementName.A).size() > 0) {
			Element a = p.getAllElements(HTMLElementName.A).get(0);
			Element img = a.getAllElements(HTMLElementName.IMG).get(0);
			String from = a.getAttributeValue("href");
			String url = img.getAttributeValue("src");
			Image image = new Image(url, from);
			image.debug = p.toString();
			images.add(image);
		}
	}
	
	return images;
}
 

import net.htmlparser.jericho.HTMLElementName; //导入依赖的package包/类
private static boolean isValidLITag(Tag tag) {
	Element parentElement=tag.getElement().getParentElement();
	if (parentElement==null) return false; // ignore LI elements without a parent
	if (parentElement.getStartTag().getUserData()!=VALID_MARKER) return false; // ignore LI elements who's parent is not valid
	return parentElement.getName()==HTMLElementName.UL || parentElement.getName()==HTMLElementName.OL; // only accept LI tags who's immediate parent is UL or OL.
}
 

import net.htmlparser.jericho.HTMLElementName; //导入依赖的package包/类
private static boolean isValidTDTHTag(Tag tag) {
	Element parentElement=tag.getElement().getParentElement();
	if (parentElement==null) return false; // ignore TD, TH elements without a parent
	if (parentElement.getStartTag().getUserData()!=VALID_MARKER) return false; // ignore TD,TH elements who's parent is not valid
	return parentElement.getName()==HTMLElementName.TR; // only accept TD,TH tags who's immediate parent is TR.
}
 

import net.htmlparser.jericho.HTMLElementName; //导入依赖的package包/类
private static boolean isValidTbodyTHeadTag(Tag tag) {
	Element parentElement=tag.getElement().getParentElement();
	if (parentElement==null) return false;
	if (parentElement.getStartTag().getUserData()!=VALID_MARKER) return false;
	return parentElement.getName()==HTMLElementName.TABLE;
}
 

import net.htmlparser.jericho.HTMLElementName; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
		
		if (args.length > 0)
			inDir = args[0];
		if (args.length > 1)
			outDir = args[1];
		else
			outDir = inDir;
		
		File f = new File(inDir);
		
		for (File doc : f.listFiles()){
		
		String sourceUrlString="file:"+inDir+doc.getName();
		
		MicrosoftConditionalCommentTagTypes.register();
		PHPTagTypes.register();
		PHPTagTypes.PHP_SHORT.deregister(); // remove PHP short tags for this example otherwise they override processing instructions
		MasonTagTypes.register();
		Source source=new Source(new URL(sourceUrlString));

		PrintStream ps = new PrintStream(outDir+doc.getName().replaceFirst(".html", "")+".txt");
		
		PrintStream ps_key = new PrintStream(outDir+doc.getName().replaceFirst(".html", "")+".key");
		
		// Call fullSequentialParse manually as most of the source will be parsed.
		source.fullSequentialParse();

//		System.out.println("Document title:");
		String title=getTitle(source);
		ps.print(title+"\n\n");
//		System.out.println("\nDocument description:");
		String description=getMetaValue(source,"description");
		ps.println(description+"\n\n");
//		System.out.println("\nDocument keywords:");
		String keywords=getMetaValue(source,"keywords");
		if (keywords != null)
			ps_key.print(keywords.replaceAll("\\s*,\\s*", "\n"));
		ps_key.close();
		
		List<Element> divElements=source.getAllElements(HTMLElementName.DIV);
		for (Element div : divElements) {
			String id=div.getAttributeValue("id");
			if (id==null) 
				continue;
			else if (id.equals("article-body-blocks")){
				String article=div.getContent().getTextExtractor().toString();
				ps.println(article);
			}
		}
		ps.close();
		}

  }
 

import net.htmlparser.jericho.HTMLElementName; //导入依赖的package包/类
private static String getTitle(Source source) {
	Element titleElement=source.getFirstElement(HTMLElementName.TITLE);
	if (titleElement==null) return null;
	// TITLE element never contains other tags so just decode it collapsing whitespace:
	return CharacterReference.decodeCollapseWhiteSpace(titleElement.getContent());
}
 

import net.htmlparser.jericho.HTMLElementName; //导入依赖的package包/类
private HtmlStripper criarHtmlStripper() {
	HtmlStripper stripper = new HtmlStripper();
	stripper.addAllowedTags(HTMLElementName.DIV, HTMLElementName.HR, HTMLElementName.FONT);
	stripper.addAllowedAttributes("size", "face", "style", "color");
	return stripper;
}