I did a statistic analysis of the recently leaked 20 million hotel reservation records and here’re some preliminary results.
Tag: MongoDB
http://docs.mongodb.org/manual/core/map-reduce/
http://docs.mongodb.org/manual/reference/command/mapReduce/
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 |
> db.lattern_money_record.mapReduce( function() { emit(this.quantity, 1) }, function(key, values) { return Array.sum(values) }, { query: {'quantity': {$gt: 500}}, out: {inline: 1} } ) { "results" : [ { "_id" : 550, "value" : 3 }, { "_id" : 570, "value" : 1 }, { "_id" : 580, "value" : 1 }, { "_id" : 583, "value" : 1 }, { "_id" : 587, "value" : 1 }, { "_id" : 600, "value" : 2 }, { "_id" : 660, "value" : 1 }, { "_id" : 700, "value" : 2 }, { "_id" : 800, "value" : 5 }, { "_id" : 900, "value" : 2 }, { "_id" : 924, "value" : 1 }, { "_id" : 949, "value" : 1 }, { "_id" : 980, "value" : 1 }, { "_id" : 990, "value" : 1 }, { "_id" : 1000, "value" : 12 } ], "timeMillis" : 36, "counts" : { "input" : 35, "emit" : 35, "reduce" : 6, "output" : 15 }, "ok" : 1, } |
The MapReduce code I used to analyze the 20 million hotel reservation records:
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 |
def get_aggregation(collection): ''' 1. Get unique set of people 2. Get most frequent users 3. Get aggregation by location of birth, age, month and day of birth ''' # Emit multiple times in mapper function: # http://docs.mongodb.org/manual/reference/command/mapReduce/ mapper = Code(''' function() { function validate_rid(id) { // From: https://gist.github.com/foxwoods/1817822 // 18位身份证号 // 国家标准《GB 11643-1999》 function rid18(id) { if(! /\d{17}[\dxX]/.test(id)) { return false; } var modcmpl = function(m, i, n) { return (i + n - m % i) % i; }, f = function(v, i) { return v * (Math.pow(2, i-1) % 11); }, s = 0; for(var i=0; i<17; i++) { s += f(+id.charAt(i), 18-i); } var c0 = id.charAt(17), c1 = modcmpl(s, 11, 1); return c0-c1===0 || (c0.toLowerCase()==='x' && c1===10); } // 15位身份证号 // 2013年1月1日起将停止使用 // http://www.gov.cn/flfg/2011-10/29/content_1981408.htm function rid15(id) { var pattern = /[1-9]\d{5}(\d{2})(\d{2})(\d{2})\d{3}/, matches, y, m, d, date; matches = id.match(pattern); y = +('19' + matches[1]); m = +matches[2]; d = +matches[3]; date = new Date(y, m-1, d); return (date.getFullYear()===y && date.getMonth()===m-1 && date.getDate()===d); } // return rid18(id) || rid15(id); try { ret = rid18(id) || rid15(id); return ret; } catch (err) { return false; } } function validateEmail(email) { // http://stackoverflow.com/questions/46155/validate-email-address-in-javascript var re = /^(([^<>()[\]\\.,;:\[email protected]\"]+(\.[^<>()[\]\\.,;:\[email protected]\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/; return re.test(email); } var str = this.CtfId; if (str && validate_rid(str)) { var prov = parseInt(str.slice(0, 2)); var year, month, day, sex; if (str.length == 15) { year = parseInt('19' + str.slice(6, 8)); month = parseInt(str.slice(8, 10)); day = parseInt(str.slice(10, 12)); sex = parseInt(str.slice(14, 15)) % 2 ? 'M' : 'F'; } else { year = parseInt(str.slice(6, 10)); month = parseInt(str.slice(10, 12)); day = parseInt(str.slice(12, 14)); sex = parseInt(str.slice(16, 17)) % 2 ? 'M' : 'F'; } var age = 2013 - year; var valid_provs = [11, 12, 13, 14, 15, 21, 22, 23, 31, 32, 33, 34, 35, 36, 37, 41, 42, 43, 44, 45, 46, 50, 51, 52, 53, 54, 61, 62, 63, 64, 65, 71, 81, 82, 91]; if (age <= 0 || age > 100 || month <=0 || month > 12 || day <= 0 || day > 31 || valid_provs.indexOf(prov) == -1) { emit('Corrupted', 1); } else { // emit('Province ' + prov, 1); // emit('Age ' + age, 1); // emit('Month ' + month, 1); // emit('Day ' + day, 1); // emit('Sex ' + sex, 1); // emit('Prov ' + prov + ' Sex ' + sex, 1); // if (this.Address && this.Address.length > 3) { // var cur_prov = this.Address.slice(0, 3); // emit('From ' + prov + ' to ' + cur_prov, 1); // } // var email = this.EMail; // if (email && validateEmail(email)) { // var idx = email.lastIndexOf('@'); // var domain = email.slice(idx + 1); // emit(domain.toLowerCase(), 1); // } if (prov == 32 && sex == 'M') { emit(str, 1); } // if (prov == 32 && sex == 'F') { // emit(str, 1); // } } } else { emit('Corrupted', 1); } }''') reducer = Code(''' function(key, values) { return Array.sum(values); }''') result = collection.map_reduce( mapper, reducer, 'aggregation', query={'CtfTp': 'ID'} ) return result |
Leave a Comment