array.p.unflat

🦋 Proposals
1

real world problem

tdameritrade has api to retrieve up to ~400 stock-quotes in a single api-call:

require("https").get(
    (
        "https://api.tdameritrade.com/v1/marketdata/quotes?" +
        "apikey=<apikey>&symbol=" +
        "AAPL%2C" +
        "AMZN%2C" +
        "GOOG%2C" +
        .... +
        "ZTS"
    ),
    function (res) {...}
)

i want a builtin, idiomatic/pythonic way to break large lists of stocks symbols into smaller chunks of 400 for the api-call.

propose elegant solution using boolean function that will delimit chunks whenever truthy:

node -e '
/*jslint devel, this*/
"use strict";
// polyfill
Array.prototype.unflat = function (fnc) {
    let aa;
    let chunk;
    let result;
    chunk = [];
    result = [chunk];
    this.forEach(function (bb, ii) {
        // create new chunk if fnc() returns truthy
        if (ii !== 0 && fnc(ii, aa, bb)) {
            chunk = [];
            result.push(chunk);
        }
        chunk.push(bb);
        aa = bb;
    });
    return result;
};


// unflat large-list in chunks of 3
let result;
result = ["fb", "aapl", "amzn", "nflx", "goog"];
result = result.unflat(function (ii) {
    return ii % 3 === 0;
});
console.log(result);
// [ [ "fb", "aapl", "amzn" ], [ "nflx", "goog" ] ]


// unflat large-list by first-letter in alphabetical-order
result = ["fb", "aapl", "amzn", "nflx", "goog"];
result.sort();
result = result.unflat(function (ignore, aa, bb) {
    return aa[0] !== bb[0];
});
console.log(result);
// [ [ "aapl", "amzn" ], [ "fb" ], [ "goog" ], [ "nflx" ] ]
'

# stdout
# [ [ 'fb', 'aapl', 'amzn' ], [ 'nflx', 'goog' ] ]
# [ [ 'aapl', 'amzn' ], [ 'fb' ], [ 'goog' ], [ 'nflx' ] ]

bikeshed name unflat

note proposed array.p.unflat is unrelated to npm-module array-unflat, in case of name-clash, alternative-names could be array.p.partition, array.p.groupby, etc...

2

It is familiar with _.partition, and _.groupBy is another similar tutorial that should return an object.

3

_.partition and _.groupBy don't seems applicable to my use-case:

segmenting large-list of 1000+ elements into smaller lists of 400-elements to overcome tdameritrade's 400-element api-call-limit.

segmenting large-lists into smaller, size-bounded chunks is common desing-pattern in parallel-processing:

#!/bin/sh
APIKEY=xxxxxxxx node -e '
/* jslint utility2:true */
(async function () {
    "use strict";
    // polyfill
    Array.prototype.unflat = function (fnc) {
        let aa;
        let chunk;
        let result;
        chunk = [];
        result = [chunk];
        this.forEach(function (bb, ii) {
            // create new chunk if fnc() returns truthy
            if (ii !== 0 && fnc(ii, aa, bb)) {
                chunk = [];
                result.push(chunk);
            }
            chunk.push(bb);
            aa = bb;
        });
        return result;
    };

    // segment 1000-element list into smaller 400-element lists for api-call
    let resultList;
    let stockList1000;
    let stockList400;
    stockList1000 = Array.from(new Array(1000), function () {
        return Math.random().toString(36).toUpperCase().slice(-4);
    });
    stockList400 = stockList1000.unflat(function (ii) {
        return ii % 400 === 0;
    });
    // debug segmented-400-element-list
    console.log(stockList400);
    // make api-call with 400-element chunks
    resultList = await Promise.all(stockList400.map(function (elem) {
        return new Promise(function (resolve) {
            require("https").get((
                "https://api.tdameritrade.com/v1/marketdata/quotes?" +
                "apikey=" + process.env.APIKEY + "&" +
                "symbol=" + elem.join("%2C")
            ), function (res) {
                let result;
                result = "";
                res.setEncoding("utf8");
                res.on("data", function (chunk) {
                    result += chunk;
                });
                res.on("end", function () {
                    resolve(result);
                });
            });
        });
    }));
    // debug parallelized api-call
    console.log(resultList);
}());
'

# stdout
# [
#   [
#     '8RE7', '1K9A', 'GXHT', '2LVH', '62OE', 'E0BG', 'ZSDM',
#     'FVEI', 'DFIQ', 'KIAK', 'LTFP', 'DFIK', 'M3JM', 'L0VF',
#     '8XAQ', '0X8P', 'ZW3I', 'EQLN', 'B12D', '1H96', 'WJCS',
#     'RY5Q', 'VQ76', '3LXW', '8J6V', 'Y3RI', '7QRD', 'KQZP',
#     'OAWD', '040C', '59B6', 'F7Q9', 'HLAJ', 'DTEB', 'GGRK',
#     'D319', 'FTE9', 'MNKI', 'FKLU', 'PG3A', 'V7DX', 'DXOL',
#     '9QPP', 'O26Q', 'BPHM', 'UO4E', 'UFSB', 'KEUK', 'STOP',
#     '7S99', '4SR4', '36RU', 'ZVJW', 'L6ID', 'AMDW', 'VDQP',
#     'BNAY', 'JU84', 'HATB', 'GFQR', '3H6J', 'LOWN', 'GDID',
#     'LPMS', 'OAGN', 'V4VR', 'KH8Y', '8IET', 'AO2S', '8MX7',
#     'UWZI', 'AUVF', '5XC7', 'IY6L', '3RPS', 'AONE', '7WJD',
#     'A7AH', 'CN0G', 'UW7A', 'BSPY', 'HLGQ', 'BEKW', '700F',
#     'MVWP', 'MRGD', 'UZXX', 'PFFF', 'J7WS', 'LSD8', 'R1RO',
#     'Y9QE', 'XG4V', 'FEFA', 'EIN3', 'ONIK', 'R6OJ', 'NIQB',
#     'MOUK', 'VJOR',
#     ... 300 more items
#   ],
#   [
#     '8WLD', '4PDS', 'ULL6', '5PIQ', '4YEC', 'LWSE', 'VBFB',
#     '9AC5', 'JZRF', 'AV1P', 'C50R', 'TJU7', 'HO83', 'JGWY',
#     '7DRB', 'DLFT', 'MECR', 'R7FP', '3WDR', 'ACG8', 'C6FH',
#     'FSGQ', 'T6VR', '2G7C', 'W4PJ', '3PYP', 'I22L', 'LAUV',
#     'UHL9', '202V', '4BIF', 'F159', '3WWD', 'RZQ5', 'R88I',
#     'T03G', 'VIXD', '1T6M', 'DWK8', '4TPO', 'IKBS', 'E975',
#     '7ZW3', 'U3CF', 'P1HS', 'QDGG', 'OIX6', 'YT1G', '6VM8',
#     'PP9T', '5HHB', 'QZ0L', '3B5O', 'QIVQ', 'DNDP', '80WH',
#     'M9YB', 'M0EO', 'CLM9', '0HAG', '44JM', 'FUV2', 'GW9O',
#     '1ITA', 'HK4T', 'AAON', 'PNBQ', 'W9EQ', 'JVAE', 'LV1R',
#     'E5EG', 'OX4P', 'E7PR', 'C3Q3', 'UW5M', '9Q5N', 'BDJY',
#     'AD4Z', 'JPKL', '8G3A', 'VK5B', 'P7GA', '1I03', 'P4BI',
#     'W0S8', 'K7HG', '6EDG', '3XIH', 'H4RS', '9HKS', 'G7TC',
#     'GZL7', 'FRA3', 'KX27', '0L3A', 'WL8E', '5A8E', 'I9I8',
#     'IBUN', 'RKPC',
#     ... 300 more items
#   ],
#   [
#     'IJBN', '946K', 'SC04', 'CT59', 'ECYV', 'UG2B', 'TQ0H',
#     '2UOZ', 'PQLC', 'K9YZ', 'GB9H', 'DBDS', '9CHU', 'MX1M',
#     '1VTL', '0KIS', '5T8H', '82XH', 'QSGG', 'Y0JQ', 'K8X5',
#     'GC2K', 'VOTI', 'O11U', 'QQ2O', 'J8EC', '635G', 'AO3Y',
#     'ZGNJ', 'G10K', '9NR8', 'DAND', '169C', '4J55', '6XIJ',
#     'W3OR', '2KLR', 'EP29', 'RW3D', 'EWUS', 'IQEO', '026M',
#     'VFRE', 'F1OH', 'GHP3', 'BSJK', 'D31K', '4H18', 'XQ4H',
#     'S3C9', 'RY1E', 'O31F', '8KNG', 'JKE9', 'IRCO', 'P0IY',
#     '7U5P', 'DV0E', '13TO', '45LL', 'TKNQ', '29LW', 'V76P',
#     '4JAD', 'W7RC', '34NY', 'N62I', 'SE2S', 'XT1K', 'TOUI',
#     'O7FD', 'TZ3F', 'GRS4', '3TC1', 'YARA', '3S2L', '1OKR',
#     'J149', '8ZAA', 'UTH9', '3L7S', 'F0JN', 'TCJA', 'HUHQ',
#     'BSSI', '9MD3', 'OGL1', 'Q1A3', 'ZIUJ', 'RFXE', 'OF9C',
#     'C2RJ', '3BEK', '5397', 'ZUDG', 'LQSR', '42Y8', 'XJL8',
#     'SSOH', 'CQCI',
#     ... 100 more items
#   ]
# ]

# [
#   '{"AONE":{"assetType":"EQUITY","assetMainType":"EQUITY",...}}',
#   '{"AAON":{"assetType":"EQUITY","assetMainType":"EQUITY",...}}',
#   '{"PQLC":{"assetType":"ETF","assetMainType":"EQUITY",...}}'
# ]
4

This is basically _.chunk from Lodash and Underscore, and I'd suggest perusing through the various proposals.

1 Like
5

k and thx. i'm not that familiar with underscore tbh (as vanilla-js developer).