From f56869feedb3741c6484dde27e36a30602beb0ce Mon Sep 17 00:00:00 2001 From: Correl Date: Tue, 14 Sep 2021 23:50:54 -0400 Subject: [PATCH] updates --- aweber/20210813142844-projects.org | 2 +- aweber/20210901164718-test.org | 1 + ...210914142000-subscriber_search_service.org | 51 +++++++++++++++ ...20210914143542-sites_subscriber_search.org | 42 +++++++++++++ ...0210914144833-broadcast_segment_search.org | 7 +++ aweber/search-components.svg | 47 ++++++++++++++ daily/2021-09-03.org | 63 +++++++++++++++++++ daily/2021-09-14.org | 12 ++++ ...ons_learned_avoiding_primitives_in_elm.org | 7 +++ 9 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 aweber/20210901164718-test.org create mode 100644 aweber/20210914142000-subscriber_search_service.org create mode 100644 aweber/20210914143542-sites_subscriber_search.org create mode 100644 aweber/20210914144833-broadcast_segment_search.org create mode 100644 aweber/search-components.svg create mode 100644 daily/2021-09-03.org create mode 100644 daily/2021-09-14.org create mode 100644 lessons_learned_avoiding_primitives_in_elm.org diff --git a/aweber/20210813142844-projects.org b/aweber/20210813142844-projects.org index 7bfda81..8cd09b8 100644 --- a/aweber/20210813142844-projects.org +++ b/aweber/20210813142844-projects.org @@ -42,7 +42,7 @@ - State "TODO" from [2021-09-01 Wed 13:44] :END: * Tracking live vs dead / removed code branches in Sites -* Search Service +* [[id:11edd6c9-b976-403b-a419-b5542ddedaae][Subscriber Search Service]] * Analytics View Service * Replace CAPI Services ** List API diff --git a/aweber/20210901164718-test.org b/aweber/20210901164718-test.org new file mode 100644 index 0000000..0214a0b --- /dev/null +++ b/aweber/20210901164718-test.org @@ -0,0 +1 @@ +#+title: Test diff --git a/aweber/20210914142000-subscriber_search_service.org b/aweber/20210914142000-subscriber_search_service.org new file mode 100644 index 0000000..16c82d6 --- /dev/null +++ b/aweber/20210914142000-subscriber_search_service.org @@ -0,0 +1,51 @@ +:PROPERTIES: +:ID: 11edd6c9-b976-403b-a419-b5542ddedaae +:END: +#+title: Subscriber Search Service + +A replacement for the current [[id:d9cb2b55-3b0e-4ab3-8369-f71ebc3cd882][Sites Subscriber Search]] and [[id:f74e335d-577f-4749-bf32-1c025795b039][Broadcast Segment Search]] implementations. + +- [[file:~/org-aweber/worklog-archive.org::#2019-08-06-tuesday-discuss-subscriber-management-architecture][file:~/org-aweber/worklog-archive.org::#2019-08-06-tuesday-discuss-subscriber-management-architecture]] +- https://jira.aweber.io/issues/?jql=project%20%3D%20CCPANEL%20AND%20component%20%3D%20%22Search%20Service%22 + +* Concerns +** Performance +** Fitness to Purpose +This service will need to fulfill the needs of both end-user subscriber searches +and segment emailing. + +* Interim Solution +** Component Diagram +#+BEGIN_SRC plantuml :file search-components.svg + database Analytics { + database Ana as Ana01 + database Ana as Ana02 + database Ana as Ana03 + } + database App + database "Results Cache" as ResultsCache + + component "Search Service" as Service { + + component Search + component Results + + Search -- Analytics + Search -- App + Search --> ResultsCache + + Results <-- ResultsCache + + } +#+END_SRC + +#+RESULTS: +[[file:search-components.svg]] + +* Resources +- [[https://confluence.aweber.io/display/AR/PostgreSQL+Backed+Search][PostgreSQL Backed Search]] (Rejected ACP) +- [[https://confluence.aweber.io/display/AR/Search+Proxy+Service][Search Proxy Service]] +- +[[https://confluence.aweber.io/display/~robink/SoT+-+ElasticSearch+Next+Steps][SoT - ElasticSearch Next Steps]]+ +- [[https://confluence.aweber.io/display/~robink/Alternative+Search+Proposal][Alternative Search Proposal]] +- [[https://confluence.aweber.io/display/AR/Search+Service+Using+Existing+Databases][Search Service Using Existing Databases]] (Approved ACP) +- [[https://confluence.aweber.io/display/AR/Search+DSL+JSON+Schema][Search DSL JSON Schema]] diff --git a/aweber/20210914143542-sites_subscriber_search.org b/aweber/20210914143542-sites_subscriber_search.org new file mode 100644 index 0000000..44daf96 --- /dev/null +++ b/aweber/20210914143542-sites_subscriber_search.org @@ -0,0 +1,42 @@ +:PROPERTIES: +:ID: d9cb2b55-3b0e-4ab3-8369-f71ebc3cd882 +:END: +#+title: Sites Subscriber Search + +* Sorting +Added: [2020-04-14 Tue 13:34] + +The current sites search code includes the following functioning code for +setting a sort order on a search based on form input: + +https://gitlab.aweber.io/CP/applications/sites/blob/52d1d944854554c5818ef9a46c8a12493599eb55/aweber_app/controllers/queries_controller.php#L386-402 +#+begin_src php :exports code :eval never + // Look up the column for the order by clause. There are no SQL column name values passed publicly. + if (!empty($this->data['SearchOrder']['SearchInput'])){ + $this->SearchInput->recursive = -1; + if ($time = $this->SearchMutex->lock($aId, '6')) { + $orderCol = $this->SearchInput->find(array('SearchInput.id' => $this->data['SearchOrder']['SearchInput'])); + $this->SearchMutex->unlock($aId, '6', $time); + } + if (!empty($orderCol['SearchInput']['column'])){ + //Case-insensitive text sorting. + // Lower text fields so that ordering is case insensitive. SearchInputs 5, 23, and 24 are actually integers, despite + // having a text search input. refs #3275 + if($orderCol['SearchInput']['input_type'] == 'text' && !in_array($orderCol['SearchInput']['id'], array(5,23,24))) { + $orderCol['SearchInput']['column'] = 'lower('.$orderCol['SearchInput']['column'].')'; + } + $this->data['SearchOrder']['column'] = $orderCol['SearchInput']['column']; + } + } +#+end_src + +- It is saved with the segment +- It is passed back to the front-end when loading a saved segment +- The =SearchCriteria= class incorporates the selected ordering and column when + building its query for a targeted search database. +- The broadcast segment service ignores the selected ordering, opting for its + own for deliverability reasons. +- All search inputs NOT in the analytics database are available for sorting (https://gitlab.aweber.io/CP/applications/sites/blob/f7ea2e9431e3ed2e694730f6446b4b3828d7c8fe/aweber_app/views/helpers/search_form.php#L54-62). +- Performance degrades with list size, likely due to memory constraints and + unindexed sort fields + (https://www.cybertec-postgresql.com/en/postgresql-improving-sort-performance/). diff --git a/aweber/20210914144833-broadcast_segment_search.org b/aweber/20210914144833-broadcast_segment_search.org new file mode 100644 index 0000000..0745a27 --- /dev/null +++ b/aweber/20210914144833-broadcast_segment_search.org @@ -0,0 +1,7 @@ +:PROPERTIES: +:ID: f74e335d-577f-4749-bf32-1c025795b039 +:END: +#+title: Broadcast Segment Search + +Performs a search using a stored segment and builds an iterable list of +recipients for a broadcast email. Implemented in the service's [[https://gitlab.aweber.io/edeliv/Applications/broadcast-segment/-/blob/master/broadcastsegment/handlers.py#L357][BroadcastHandler]]. diff --git a/aweber/search-components.svg b/aweber/search-components.svg new file mode 100644 index 0000000..e398c7e --- /dev/null +++ b/aweber/search-components.svg @@ -0,0 +1,47 @@ +AnalyticsSearch ServiceAnaAnaAnaSearchResultsAppResults Cache \ No newline at end of file diff --git a/daily/2021-09-03.org b/daily/2021-09-03.org new file mode 100644 index 0000000..1be6f20 --- /dev/null +++ b/daily/2021-09-03.org @@ -0,0 +1,63 @@ +:PROPERTIES: +:ID: 30330ca0-3a44-4b56-8bce-6f6cce9ab115 +:END: +#+title: 2021-09-03 + +* CP crashes +Looking into [[id:57ee2f00-9bcd-4e0f-8a77-ae1f2d4cda89][Control Panel]] OOM issues in Kubernetes. + +Average of ~138mb memory usage per request. + +#+begin_example + ps aufx |grep apache | awk '{print "cat /proc/" $1 "/statm"}' | sh | grep -v open | awk '{print $0}' +#+end_example + +Memory limit should account for the 256MB APC cache. + +#+CAPTION: Josh Benner doing math on likely requirements +#+begin_quote +rps = 32 +ram per worker = 150 MB +minimum workers = rps => 32 +num workers = minimum workers * 2 => 64 +desired pod count = 8 +workers per pod = num workers / desired pod count => 8 +apc ram = 256 MB +ram per pod = ram per worker * workers per pod + apc ram => 1,456 MB + +rps = 32 +ram per worker = 150 MB +minimum workers = rps => 32 +num workers = minimum workers * 2 => 64 +desired pod count = 8 +workers per pod = num workers / desired pod count * 2 => 16 +apc ram = 256 MB +ram per pod = ram per worker * workers per pod + apc ram => 2,656 MB + +Making assumptions on baseline requests per second and duration of request. + +active workers = 20 * 8 => 160 +apc ram = 256 MB +ram per worker = 150 MB +pod count = 8 +workers per pod = active workers / pod count => 20 +ram per pod = workers per pod * ram per worker => 3,000 MB + +active workers = 20 * 8 * 2 => 320 +apc ram = 256 MB +ram per worker = 150 MB +pod count = 20 +workers per pod = active workers / pod count => 16 +ram per pod = workers per pod * ram per worker => 2,400 MB +#+end_quote + +Actions taken: +- Made per-pod worker count configurable via consul (easier to change if needed) +- Tuned pod worker counts and memory allocations based on memory usage metrics (helps avoid OOMKills) +- Removed the k8s liveness probe (avoids k8s-kills when loaded) +- Revised the k8s readiness probe to just check TCP socket availability (avoids removing pod from rotation during load) +- Load-tested these changes in staging: confirmed that pods stay up under load, and scaling more readily mitigates pod +- Monitoring added to draw attention to load-induced symptoms of control-panel pods. +- Auto-scaling will be investigated next week as an additional load mitigation tool. + +(details: 32 workers per pod, 1Gi RAM per pod (for now), 8 pods) diff --git a/daily/2021-09-14.org b/daily/2021-09-14.org new file mode 100644 index 0000000..0e31a42 --- /dev/null +++ b/daily/2021-09-14.org @@ -0,0 +1,12 @@ +:PROPERTIES: +:ID: d6eaaf94-d72b-4e8e-b76c-318c1fab5ada +:END: +#+title: 2021-09-14 +* Grooming with Time Estimates +Is this fundamentally any better than tracking velocity of arbitrary point +completion? It seems it'd be worse, in that time-based estimates can give a +false sense of actual time to completion. + + +- How does time estimation affect sprint capacity expectations? +- Will this blur the lines between estimate and commitment? diff --git a/lessons_learned_avoiding_primitives_in_elm.org b/lessons_learned_avoiding_primitives_in_elm.org new file mode 100644 index 0000000..7940ab8 --- /dev/null +++ b/lessons_learned_avoiding_primitives_in_elm.org @@ -0,0 +1,7 @@ +:PROPERTIES: +:ID: 67c1db33-2a17-4bcb-8b15-1b5e108ff9f6 +:ROAM_REFS: https://thoughtbot.com/blog/lessons-learned-avoiding-primitives-in-elm +:END: +#+title: Lessons Learned: Avoiding Primitives in Elm + +Discusses using specialized wrapper types in Elm.