import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
private void processTweet(Tweet tweet) {
        String lang = tweet.getLanguageCode();
        //String text = tweet.getText();
        // filter non-English tweets:
        if (!"en".equals(lang)) {
            return;
        }

        //Set<String> hashTags = hashTagsFromTweet(text);

        // filter tweets without hashtags:
//        if (hashTags.isEmpty()) {
//            return;
//        }

        log.info("\n Tweet: " + tweet.getText() + "\n");

        dataService.saveTweet(tweet);

    }
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("/news")
public Map<Tweet, String> news() throws InvalidInputException {
    List<Trend> someTrends = trendSearch.getTrends(1);

    Map<Tweet, String> allClassification = new HashMap<>();

    for (Trend iterator : someTrends) {
        List<Tweet> tweets = twitterScrape.scrape(iterator.getName());

        for (Tweet iteratorTweet : tweets) {
            if (iteratorTweet.getLanguageCode().equals("en") == true) {
                String classification = new Request().classify(iteratorTweet);

                allClassification.put(iteratorTweet, classification);
            }
        }
    }
    return allClassification;
}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
@Bean
CommandLineRunner tweets(Logger logger, TwitterTemplate twitterTemplate, JdbcTemplate jdbcTemplate) {
    return args -> {
        String screenName = "hintjens";
        int pageSize = 200;
        List<Tweet> userTimeline = twitterTemplate.timelineOperations()
                .getUserTimeline(screenName, pageSize);
        long maxId;
        while (userTimeline.size() != 0 ) {
            maxId = userTimeline.stream()
                    .map(Tweet::getId)
                    .reduce((tweet, tweet2) -> tweet < tweet2 ? tweet : tweet2)
                    .orElse(0L);
            userTimeline = twitterTemplate.timelineOperations()
                    .getUserTimeline(screenName, pageSize, 0L, maxId - 1);
            addAllTweets(jdbcTemplate, userTimeline);
            logger.info("added " + userTimeline.size() + " records.");
        }
    };
}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
private TweetEntity save(final Tweet tweetEntity, final UserEntity user) {
	TweetEntity tweet = new TweetEntity();
	tweet.setCreatedAt(tweetEntity.getCreatedAt());
	tweet.setLanguageCode(tweetEntity.getLanguageCode());
	tweet.setSource(tweetEntity.getSource());
	tweet.setText(tweetEntity.getText());
	tweet.setUser(user);

	if (tweetRepository != null) {
		tweet = tweetRepository.save(tweet);
		System.out.println(tweetEntity.getText());
		return tweet;
	}

	return null;
}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
@GetMapping(path = "tweet")
public String searchTwitter(Model model, @RequestParam String search) {
	
	int count = 200;
	
    SearchResults results = twitterTemplate.searchOperations().search(
    	    new SearchParameters(search)
    	        .resultType(SearchParameters.ResultType.RECENT)
    	        .count(count));
    
    List<Tweet> tweets = results.getTweets();        
    model.addAttribute("tweets", tweets);
    model.addAttribute("count", count);
    model.addAttribute("search", search);
    
    return "search";
}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
@Transactional
public TimelineMusic storeUserTimelinePiece(User user) {
    if (user == null) {
        return null;
    }

    SocialAuthentication auth = userDao.getTwitterAuthentication(user);

    if (auth == null) {
        return null;
    }

    Twitter twitter = provider.getApi(auth.getToken(), auth.getSecret());
    List<Tweet> tweets = twitter.timelineOperations().getUserTimeline(200);

    TimelineMusic meta = getUserTimelinePiece(tweets);
    meta.setTwitterHandle(twitter.userOperations().getScreenName());
    meta.setUser(user);

    meta = pieceDao.persist(meta);
    return meta;
}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
/**
 * Gets the tempo, depending on the rate of tweeting
 *
 * @param tweets
 * @return tempo
 */
private Tempo getTempo(List<Tweet> tweets, TimelineMusic meta) {
    long totalSpacingInMillis = 0;
    Tweet previousTweet = null;
    for (Tweet tweet : tweets) {
       if (previousTweet != null) {
           totalSpacingInMillis += Math.abs(previousTweet.getCreatedAt().getTime() - tweet.getCreatedAt().getTime());
       }
       previousTweet = tweet;
    }

    double averageSpacing = totalSpacingInMillis / (tweets.size() - 1);
    meta.setAverageSpacing(averageSpacing);

    if (averageSpacing > 3 * DateTimeConstants.MILLIS_PER_DAY) { //once every three days
        return Tempo.VERY_SLOW;
    } else if (averageSpacing > 1.5 * DateTimeConstants.MILLIS_PER_DAY) { // more than once every 1.5 days
        return Tempo.SLOW;
    } else if (averageSpacing > 16 * DateTimeConstants.MILLIS_PER_HOUR) { // more than once every 16 hours
        return Tempo.MEDIUM;
    } else if (averageSpacing > 4 * DateTimeConstants.MILLIS_PER_HOUR) { // more than once every 4 hours
        return Tempo.FAST;
    } else {
        return Tempo.VERY_FAST;
    }
}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
/**
 * Sentiment determines major or minor scale (or lydian/dorian ~= neutral)
 * Average length of tweets determines pentatonic or heptatonic
 *
 * @param tweets
 * @return scale
 */
private Scale getScale(List<Tweet> tweets, TimelineMusic meta) {
    Set<String> documents = new HashSet<>();
    for (Tweet tweet : tweets) {
        documents.add(tweet.getText());
    }
    SentimentResult sentiment = sentimentAnalyzer.getSentiment(documents, meta);

    double totalLength = 0;
    for (String document : documents) {
        totalLength += document.length();
    }
    double averageLength = totalLength / documents.size();

    meta.setSentiment(sentiment);
    meta.setAverageLength(averageLength);

    if (sentiment == SentimentResult.POSITIVE) {
        return averageLength < 40 ? Scale.MAJOR_PENTATONIC : Scale.MAJOR;
    } else if (sentiment == SentimentResult.NEGATIVE) {
        return averageLength < 40 ? Scale.MINOR_PENTATONIC : Scale.MINOR;
    }
    // choose rarer scales for neutral tweets
    return Chance.test(50) ? Scale.LYDIAN : Scale.DORIAN;
}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
@Override
public void run() {
    while (true) {
        try {
            Tweet tweet = queue.take();
            processTweet(tweet);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
/**
* 
* @param twitterUser
* @return
*/
public List <Tweet> getUserTimeline(String twitterUser) {
    TimelineOperations timelineOps = twitter.timelineOperations();
    List<Tweet> tweets = timelineOps.getUserTimeline("@" + twitterUser);

    return tweets;
}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
public List<Tweet> scrape(String term) throws InvalidInputException{
    if(term == null || term.length()==0)
        throw new InvalidInputException();

    List<Tweet> tweetList = twitterTemplate.searchOperations().search(term).getTweets();

    tweetList.forEach(tweetEntity -> {
        twitter.entity.Tweet tweet = new twitter.entity.Tweet();
        tweet.setText(tweetEntity.getText());
        tweetService.save(tweet);
    });
    return tweetList;
}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
public String classify(Tweet tweet) {
    MultiValueMap<String, String> allArguments= new LinkedMultiValueMap<>();

    allArguments.add("Date", tweet.getCreatedAt().toString());
    allArguments.add("Tweet_Text", tweet.getText());
    allArguments.add("Tweet_Id", tweet.getIdStr());
    allArguments.add("User_Id", Long.toString(tweet.getFromUserId()));
    allArguments.add("User_Name", tweet.getUser().getName());
    allArguments.add("User_Screen_Name", tweet.getUser().getScreenName());
    allArguments.add("Retweets", Integer.toString(tweet.getRetweetCount()));
    allArguments.add("Favorites", Integer.toString(tweet.getFavoriteCount()));

    HttpHeaders headers = new HttpHeaders();
    headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);

    HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<MultiValueMap<String, String>>(allArguments, headers);

    ResponseEntity<String> response = restTemplate.postForEntity( requestURL, request , String.class );

    return response.getBody();
}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("/tweets/{tweetTerm}")
public List<Tweet> search(@PathParam("tweetTerm") String tweetTerm) {
    List<Tweet> tweetList = null;
    try {
        tweetList = twitterScrape.scrape(tweetTerm);
    } catch (InvalidInputException e) {
        e.printStackTrace();
    }
    return tweetList;
}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
@Test(expected = InvalidInputException.class)
public final void trendSearchAndScrapeWithSentiment_tweetIsNull_ShouldThrowException() throws InvalidInputException {
    List<Trend> trends = trendSearch.getTrends(WORLD_WOEID).subList(0, 10);
    assertNotNull(trends);
    for (Trend trend : trends) {
        List<Tweet> tweets = twitterScrape.scrape(trend.getName()).subList(0, 10);
        assertNotNull(tweets);

        for (Tweet tweet : tweets) {
            int sentiment = sentimentAnalysis.calculateSentiment(null);
        }

    }
}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
@Test
public final void trendSearchAndScrape() throws InvalidInputException {
    List<Trend> trends = trendSearch.getTrends(WORLD_WOEID).subList(0, 5);//get some trends
    assertNotNull(trends);
    for (Trend trend : trends) {
        List<Tweet> tweets = twitterScrape.scrape(trend.getName()).subList(0, 5);
        assertNotNull(tweets);
    }
}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
@Override
@HystrixCommand(commandKey = "GetTweets", fallbackMethod = "noResults", commandProperties = {
        @HystrixProperty(name = "execution.isolation.thread.timeoutInMilliseconds", value = "5000")
})
public Collection<String> search(String q, int pageSize) {
    SearchResults results = twitter.searchOperations().search(q, pageSize);
    return results.getTweets().stream()
            .map(Tweet::getUnmodifiedText)
            .collect(toSet());
}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
public Response(SearchResults results, int count) {
	for (int i = 0; i < results.getTweets().size(); i++) {
		if (statuses == null) this.statuses = new ArrayList<Tweet>(); 
		this.statuses.add(results.getTweets().get(i));
		if (i >= count - 1) break;
	}
	search_metadata = results.getSearchMetadata();
}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
@Override
@Transactional
public void processTweet(final Tweet tweetEntity) {

	final UserEntity user = findOrCreate(tweetEntity.getUser());
	if (user != null) {
		save(tweetEntity, user);
	}

	saveEs(tweetEntity);

}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
private TweetEsEntity saveEs(final Tweet tweetEntity) {
	final TwitterProfile twitterProfile = tweetEntity.getUser();

	final UserEsEntity user = new UserEsEntity();
	user.setCreatedDate(twitterProfile.getCreatedDate());
	user.setDescription(twitterProfile.getDescription());
	user.setLocation(twitterProfile.getLocation());
	user.setName(twitterProfile.getName());
	user.setProfileUrl(twitterProfile.getProfileUrl());
	user.setProfilImageUrl(twitterProfile.getProfileImageUrl());
	user.setScreenName(twitterProfile.getScreenName());
	user.setTwitterId(twitterProfile.getId() + "");

	TweetEsEntity esTweet = new TweetEsEntity();
	esTweet.setCreatedAt(tweetEntity.getCreatedAt());
	esTweet.setLanguageCode(tweetEntity.getLanguageCode());
	esTweet.setSource(tweetEntity.getSource());
	esTweet.setText(tweetEntity.getText());
	esTweet.setUser(user);

	if (tweetEsRepository != null) {
		esTweet = tweetEsRepository.save(esTweet);
		return esTweet;
	}

	return null;
}
 

import org.springframework.social.twitter.api.Tweet; //导入依赖的package包/类
@Override
public void run() {
	while (true) {
		try {
			final Tweet tweet = queue.take();
			processTweet(tweet);
		} catch (final InterruptedException e) {
			e.printStackTrace();
		}
	}
}