在上篇文章Java数据爬取——爬取携程酒店数据(一)爬取所有地区后，继续根据地区数据爬取酒店数据

1.首先思考怎样根据地域获取地域酒店信息，那么我们看一下携程上是怎样获得的。

还是打开http://hotels.ctrip.com/domestic-city-hotel.html 这个地址，随便点击一个地区进去(这里我选取澳门作为示例)，点击第二页数据，很高兴发现

2.模拟请求，获取数据

从这里可以发现，是POST请求，请求的body中有很多参数，

所以先封装一下参数，经过测试，有些参数可以不用传递，这里写了一个方法，模拟请求

public String getHotelListString(HotelCity city, String page){

HashMap params = new HashMap();

params.put("__VIEWSTATEGENERATOR", "DB1FBB6D");

params.put("cityName", city.getCityName());

// params.put("StartTime", "2016-11-24");

// params.put("DepTime", "2016-11-25");

params.put("txtkeyword", "");

params.put("Resource", "");

params.put("Room", "");

params.put("Paymentterm", "");

params.put("BRev", "");

params.put("Minstate", "");

params.put("PromoteType", "");

params.put("PromoteDate", "");

params.put("operationtype", "NEWHOTELORDER");

params.put("PromoteStartDate", "");

params.put("PromoteEndDate", "");

params.put("OrderID", "");

params.put("RoomNum", "");

params.put("IsOnlyAirHotel", "F");

params.put("cityId", city.getCityId());

params.put("cityPY", city.getPinyin());

// params.put("cityCode", "1853");

// params.put("cityLat", "22.1946");

// params.put("cityLng", "113.549");

params.put("positionArea", "");

params.put("positionId", "");

params.put("keyword", "");

params.put("hotelId", "");

params.put("htlPageView", "0");

params.put("hotelType", "F");

params.put("hasPKGHotel", "F");

params.put("requestTravelMoney", "F");

params.put("isusergiftcard", "F");

params.put("useFG", "F");

params.put("HotelEquipment", "");

params.put("priceRange", "-2");

params.put("hotelBrandId", "");

params.put("promotion", "F");

params.put("prepay", "F");

params.put("IsCanReserve", "F");

params.put("OrderBy", "99");

params.put("OrderType", "");

params.put("k1", "");

params.put("k2", "");

params.put("CorpPayType", "");

params.put("viewType", "");

// params.put("checkIn", "2016-11-24");

// params.put("checkOut", "2016-11-25");

params.put("DealSale", "");

params.put("ulogin", "");

params.put("hidTestLat", "0%7C0");

// params.put("AllHotelIds", "436450%2C371379%2C396332%2C419374%2C345805%2C436553%2C425997%2C436486%2C436478%2C344977%2C5605870%2C344983%2C371396%2C344979%2C2572033%2C699384%2C425795%2C419823%2C2010726%2C5772619%2C1181591%2C2005951%2C345811%2C371381%2C371377");// TODO

params.put("psid", "");

params.put("HideIsNoneLogin", "T");

params.put("isfromlist", "T");

params.put("ubt_price_key", "htl_search_result_promotion");

params.put("showwindow", "");

params.put("defaultcoupon", "");

params.put("isHuaZhu", "False");

params.put("hotelPriceLow", "");

params.put("htlFrom", "hotellist");

params.put("unBookHotelTraceCode", "");

params.put("showTipFlg", "");

// params.put("hotelIds", "436450_1_1,371379_2_1,396332_3_1,419374_4_1,345805_5_1,436553_6_1,425997_7_1,436486_8_1,436478_9_1,344977_10_1,5605870_11_1,344983_12_1,371396_13_1,344979_14_1,2572033_15_1,699384_16_1,425795_17_1,419823_18_1,2010726_19_1,5772619_20_1,1181591_21_1,2005951_22_1,345811_23_1,371381_24_1,371377_25_1");// TODO

params.put("markType", "1");

params.put("zone", "");

params.put("location", "");

params.put("type", "");

params.put("brand", "");

params.put("group", "");

params.put("feature", "");

params.put("equip", "");

params.put("star", "");

params.put("sl", "");

params.put("s", "");

params.put("l", "");

params.put("price", "");

params.put("a", "0");

params.put("keywordLat", "");

params.put("keywordLon", "");

params.put("contrast", "0");

params.put("page", page);

params.put("contyped", "0");

params.put("productcode", "");

String result = HttpUtil.getInstance().httpPost(hotelUrl, params);

// 数据中有转义符直接转JSON报错，所以这里重新拼接所需要的JSON数据

String tempHotel = result.substring(result.indexOf("hotelPositionJSON")-1, result.length());

// 确保截取到indexOf("biRecord"), 减2是因为需要]符号

String hotelArray = tempHotel.substring(0, tempHotel.indexOf("biRecord") - 2);

String tempTotalCount = result.substring(result.indexOf("hotelAmount")-1, result.length());

String totalCount = tempTotalCount.substring(0, tempTotalCount.indexOf(","));

StringBuffer sb = new StringBuffer();

sb.append("{");

sb.append(totalCount);

sb.append(",");

sb.append(hotelArray);

sb.append("}");

return sb.toString().replace("\\", "");

}

说明：这个方法对返回结果进行了操作，直接把hotelPositionJSON和hotelAmount的内容提取了出来(因为直接转换时发现数据中含有转义符导致转为JSON失败)，操作后的数据格式为：

{

"hotelAmount": 11265,

"hotelPositionJSON": [ { "id": "6297824", "name": "北京浣川招待所", "lat": "39.918086", "lon": "116.427508", "url": "/hotel/6297824.html?isFull=F#ctm_ref=hod_sr_map_dl_txt_1", "img": "http://pic.c-ctrip.com/hotels110127/hotel_example.jpg", "address": "东城区东城区外交部街46号-1。 ( 北京站、建国门地区)", "score": "0.0", "dpscore": "0", "dpcount": "0", "star": "hotel_diamond01", "stardesc": "携程用户评定为1钻", "shortName": "", "isSingleRec": "false"}, { "id": "6298279", "name": "北京怀柔喇叭沟门海燕农家院", "lat": "40.956471", "lon": "116.513108", "url": "/hotel/6298279.html?isFull=F#ctm_ref=hod_sr_map_dl_txt_2", "img": "http://pic.c-ctrip.com/hotels110127/hotel_example.jpg", "address": "怀柔区黄甸子村。 ( 怀柔风景区)", "score": "0.0", "dpscore": "0", "dpcount": "0", "star": "hotel_diamond01", "stardesc": "携程用户评定为1钻", "shortName": "", "isSingleRec": "false"} ]}

3.循环遍历城市数据，查询每个城市的酒店即可

long startTime = System.currentTimeMillis();

HotelCitySpider citySpider = new HotelCitySpider();

HotelSpider spider = new HotelSpider();

List cities = citySpider.getHotelCities();

long getCityTime = System.currentTimeMillis();

System.out.println("获取城市所用时间(ms):" + (getCityTime - startTime));

spider.createTable();

for (HotelCity hotelCity : cities) {

spider.saveHotels(hotelCity, spider.getHotelList(hotelCity));

}

long saveHotelTime = System.currentTimeMillis();

System.out.println("获取酒店并存储所用时间(ms):" + (saveHotelTime - startTime));

这里我仍然将数据存储到数据库中

4.批量插入数据

/** * 保存每个城市的酒店列表 *@param city *@param hotels */

public void saveHotels(HotelCity city, List hotels) {

for (Hotel hotel : hotels) {

StringBuffer insert_sql = new StringBuffer();

insert_sql.append("insert into ctrip_hotel "

+ "(hotel_id, city_id, city_name, name, lat, lon, url, img, address, score, dpscore, dpcount, star, stardesc, shortName, isSingleRec) values (");

insert_sql.append("'" + hotel.getId() + "'");

insert_sql.append(", " + city.getCityId());

insert_sql.append(", '" + city.getCityName() + "'");

insert_sql.append(", '" + hotel.getName() + "'");

insert_sql.append(", " + hotel.getLat());

insert_sql.append(", " + hotel.getLon());

insert_sql.append(", '" + hotel.getUrl() + "'");

insert_sql.append(", '" + hotel.getImg() + "'");

insert_sql.append(", '" + hotel.getAddress() + "'");

insert_sql.append(", " + hotel.getScore());

insert_sql.append(", " + hotel.getDpscore());

insert_sql.append(", " + hotel.getDpcount());

insert_sql.append(", '" + hotel.getStar() + "'");

insert_sql.append(", '" + hotel.getStardesc() + "'");

insert_sql.append(", '" + hotel.getShortName() + "'");

insert_sql.append(", " + hotel.getIsSingleRec() + ")");

try {

preparedStatement = conn.prepareStatement(insert_sql.toString());

preparedStatement.execute();

} catch (Exception e) {

e.getMessage();

continue;

}

}

}

原本想批量插入，但是数据有重复的hotel_id，preparedStatement.executeBatch()会报错，所以还是一条条插入。

github源码地址 https://github.com/jianiuqi/CTripSpider