From c12c9d54c1a7bfda4145fce455d328fcdfe61dea Mon Sep 17 00:00:00 2001 From: Bennie Rosas Date: Wed, 22 Oct 2025 18:28:04 -0500 Subject: [PATCH 1/7] maybe math wip --- math/.gitignore | 34 ++-- math/Dockerfile | 6 +- math/README.md | 146 +++++++++++++---- math/dev.env.sh | 5 + math/doc/commands.md | 240 ++++++++++++++++++++++++++++ math/doc/configuration.md | 190 ++++++++++++++++++++-- math/doc/update.md | 153 ++++++++++++++++++ math/src/polismath/meta/metrics.clj | 11 +- 8 files changed, 718 insertions(+), 67 deletions(-) create mode 100755 math/dev.env.sh create mode 100644 math/doc/commands.md create mode 100644 math/doc/update.md diff --git a/math/.gitignore b/math/.gitignore index a8c560ea1d..3ff3c02533 100644 --- a/math/.gitignore +++ b/math/.gitignore @@ -1,24 +1,30 @@ -/target -/lib -/classes -/checkouts -pom.xml -pom.xml.asc -*.jar -*.class +.calva/ +.cpcache +.env +.gorilla-port .lein-deps-sum +.lein-env .lein-failures +.lein-git-deps/debug-repl +.lein-git-deps/hiphip .lein-plugins .lein-repl-history +.nrepl-port +*.class +*.jar *.swo +/checkouts +/classes +/lib +/target +data db/ -.lein-env -.nrepl-port +errorconv* +old +pom.xml +pom.xml.asc scratch src/polismath/play.clj -old -.gorilla-port -.lein-git-deps/hiphip wiki .lein-git-deps/debug-repl data @@ -55,4 +61,4 @@ ENV/ .ipynb_checkpoints */.ipynb_checkpoints/* -real_data \ No newline at end of file +real_data diff --git a/math/Dockerfile b/math/Dockerfile index 85b1065502..064c3d8c01 100644 --- a/math/Dockerfile +++ b/math/Dockerfile @@ -1,11 +1,15 @@ FROM docker.io/clojure:temurin-17-tools-deps WORKDIR /app -COPY . . RUN curl -L -o /app/dd-java-agent.jar 'https://dtdg.co/latest-java-tracer' # Install clojure and fetch dependencies +# Copy deps.edn first and fetch dependencies +COPY deps.edn . RUN clojure -A:dev -P +# Then copy the rest of the source code +COPY . . + CMD ["./bin/run"] diff --git a/math/README.md b/math/README.md index 85d5090fec..b7efd5eb25 100644 --- a/math/README.md +++ b/math/README.md @@ -9,6 +9,109 @@ As with any Lisp, Clojure development typically revolves around the interactive It's recommended that you use an editor plugin to connect to this REPL, letting you execute and experiment with code as you write it, display documentation, and debug all from the comfort of your favorite text editor. (See [Calva](https://marketplace.visualstudio.com/items?itemName=betterthantomorrow.calva) for VS, [Fireplace](https://github.com/tpope/vim-fireplace) for Vim, [Cider](https://docs.cider.mx/cider/index.html) for Emacs, [Cursive](https://cursive-ide.com/) for IDEA, etc). +## Quickstart Guide + +### Prerequisites + +- [Clojure](https://clojure.org/guides/getting_started) installed +- Docker and Docker Compose (recommended for simplest setup) +- Alternatively: PostgreSQL client (`postgresql`, `postgresql-client`) if not using Docker +- A PostgreSQL database instance (either local or remote) + +### Setup with Docker (Recommended) + +1. **Clone the repository**: + + ```sh + git clone https://github.com/compdemocracy/polis.git + cd polis + ``` + +2. **Configure environment variables**: + Create a `.env` file in the root directory with at least: + + ```sh + DATABASE_URL=postgres://username:password@hostname:port/database + ``` + +3. **Start the system with Docker Compose**: + + ```sh + docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile postgres up + ``` + + If this is your first time or you've made changes to Docker files: + + ```sh + docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile postgres up --build + ``` + +4. **Connect to the REPL**: + The nREPL server will be available on port `18975`. Connect to it using your favorite editor's Clojure plugin. + +### Setup without Docker + +1. **Clone the repository**: + + ```sh + git clone https://github.com/compdemocracy/polis.git + cd polis/math + ``` + +2. **Configure environment variables**: + Set at least: + + ```sh + export DATABASE_URL=postgres://username:password@hostname:port/database + ``` + +3. **Start a development REPL**: + + ```sh + clojure -M:dev + ``` + + This starts an nREPL server but doesn't start the polling system. You can manually start it with `(runner/run!)` from within the REPL. + +### Running Commands + +The system provides several commands you can run: + +```sh +# Get help +clojure -M:run --help + +# Export a conversation +clojure -M:run export -f .zip + +# Update a specific conversation +clojure -M:run update -Z + +# Run the full system (poller plus task processing) +clojure -M:run full +``` + +If using Docker: + +```sh +docker compose run math clojure -M:run +``` + +### Running Tests + +```sh +# Run all tests +clojure -M:test + +# Or from within a REPL: +(require '[test-runner]) +(test-runner/-main) +``` + +For more information about configuration options, see [the configuration documentation](doc/configuration.md). + +## Docker Setup Details + The root directory of this codebase contains docker compose infrastructure for running a complete development environment. In addition to running the rest of the system's components (database, server, client build processes, etc), it also runs the math component in tandem with an embedded nREPL. This allows you to connect to and evaluate code from within the running math worker. @@ -30,13 +133,9 @@ To get a sense for how various parts of these system can be used, take a look at If you're not familiar with Clojure and want a fun crash course, I highly recommend [Clojure for the Brave and True](https://www.braveclojure.com/), a delightful introduction to the language. -### Without docker compose? - -If you're not using the docker compose infrastructure, you can run `clj -M:dev` to get nREPL going, but this will not start math worker's processing queue (or obviously any other parts of the system). -This may be preferable if you don't need the whole system running for whatever task you're working on. -You can always manually start the polling system by manually running `(runner/run!)`, as described below, as long as you have the `DATABASE_URL` environment variable pointing to a database (see [`doc/configuration.md`](doc/configuration.md)) +## Development Workflow -## Starting and stopping the system +### Starting and Stopping the System This application uses Stuart Sierra's Component library for REPL-reloadability. Sometimes, (e.g.) evaluating a new definition of an existing function will be picked up by the system immediately without any further work. @@ -50,27 +149,6 @@ While this setup is nice from the perspective of system reloadability, Stuart's This ends up being somewhat annoying from the perspective of interactive development, as it requires grabbing the corresponding component out of the `runner/system` map, and passing that to the function in question. We'll soon be switching to [Mount](https://github.com/tolitius/mount) over Component, for more automated reloadability, and less hassle passing around system/component objects (see [#1056](https://github.com/compdemocracy/polis/issues/1056)). -## Running commands - -There are also a number of commands which can be run, either locally or with `docker compose run math ...`, from the root of the monorepo: - -* `clojure -M:run --help` - print run command help (noisy; sorry) -* `clojure -M:run export -f .zip` - export the conversation at `` to the given filename -* `clojure -M:run update -Z ` - update a particular conversation -* `clojure -M:run full` - run a full system (poller plus auxiliary task processing) -* etc. - -If you are exporting data, you will need to run `docker compose` here with the same `-f docker-compose.yml -f docker-compose.dev.yml --profile postgres` options as described above, so that the directory (volume) mounting in the dev configuration file will mirror the generated files over onto your local filesystem. - -## Tests - -You can run tests by executing `clojure -M:test`. - -Since Clojure is slow to start though, you may find it easier to run the `test-runner/-main` function (located at [`test/test_runner.clj`](test/test_runner.clj)) from within your nREPL process. -There is an example of this in the [`dev/user.clj`](dev/user.clj) file mentioned above. -There are rough units tests for most of the basic math things, and one or two higher level integration tests (presently broken). -We're looking forward to setting up `clojure.spec` and some generative testing for more thorough coverage if this is something that excites you! - ## Architecture The system is designed around a polling mechanism which queries the database at a regular interval for new votes and moderation status. @@ -85,13 +163,21 @@ You can see the conversation manager implementation at [`src/polismath/conv_man. The system's logging level defaults to `:warn` but can be configured in several ways: -1. Set the `LOGGING_LEVEL` environment variable to change from the default `:warn` level: +1. Set the logging level environment variable: + - When running directly: use `LOGGING_LEVEL` + - When using Docker Compose: use `MATH_LOG_LEVEL` (Docker translates this to `LOGGING_LEVEL` internally) ```bash + # When using Docker Compose: MATH_LOG_LEVEL=info docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile postgres up + + # When running directly: + export LOGGING_LEVEL=info + clojure -M:dev ``` 2. Modify the logging level in `resources/config.edn` + 3. Set the level at runtime via the REPL: ```clojure @@ -111,8 +197,8 @@ The individual `Dockerfile`s that make up this infrastructure can currently be u Nonetheless, if you wish to run this part of the system directly on a machine (outside of docker), the only requirements are that you: -* [install Clojure](https://clojure.org/guides/getting_started). -* Setup the postgresql client (`sudo apt-get install postgresql postgresql-client` on ubuntu). +- [install Clojure](https://clojure.org/guides/getting_started). +- Setup the postgresql client (`sudo apt-get install postgresql postgresql-client` on ubuntu). If you go this route you will want to take a look at the (see [`doc/configuration.md`](doc/configuration.md)). diff --git a/math/dev.env.sh b/math/dev.env.sh new file mode 100755 index 0000000000..daae5c9ec2 --- /dev/null +++ b/math/dev.env.sh @@ -0,0 +1,5 @@ +#! /bin/bash + +export DATABASE_IGNORE_SSL=true +export DATABASE_URL=postgres://postgres:oiPorg3Nrz0yqDLE@localhost:5432/polis-dev +export MATH_ENV=dev diff --git a/math/doc/commands.md b/math/doc/commands.md new file mode 100644 index 0000000000..23646c3a24 --- /dev/null +++ b/math/doc/commands.md @@ -0,0 +1,240 @@ +# Polismath Commands Guide + +This document provides detailed information about the commands and options available for running the Polismath system. + +## Clojure CLI Commands + +The Polismath project uses several Clojure CLI commands for different purposes: + +### Common Command Flags + +- **`-A`**: Specifies an alias for dependency resolution and classpath configuration +- **`-M`**: Specifies an alias AND runs the `-main` function in the namespace specified by the alias +- **`-P`**: Prepares/downloads dependencies without running any code (a.k.a. "prep") + +### Common Commands Used in the Project + +- **`clojure -A:dev -P`**: Used in the Dockerfile to download dependencies specified by the `:dev` alias without running code +- **`clojure -M:dev`**: Starts an nREPL server with development dependencies +- **`clojure -M:run `**: Runs the specified subcommand through the runner system +- **`clojure -M:test`**: Runs the test suite + +## Available Subcommands + +The Polismath system provides several subcommands that run different components or operations: + +| Subcommand | System Type | Description | +|------------|-------------|-------------| +| `update-all` | base-system | Updates all conversations in the database | +| `update` | base-system | Updates a specific conversation by ID | +| `poller` | poller-system | Runs the polling system to watch for new votes and moderation | +| `tasks` | task-system | Runs the task system for auxiliary jobs | +| `full` | full-system | Runs both poller and task systems together | +| `simulator` | simulator-system | Runs a simulation system (for development/testing) | +| `export` | export-system | Exports conversation data to files | + +These subcommands are defined in the `polismath.runner` namespace, with execution paths determined by the `-main` function. + +## Command-Line Options + +### General Options (Most Commands) + +```txt +-r, --recompute Recompute conversations from scratch instead of starting from most recent values +-h, --help Print help and exit +-z, --zid ZID ZID (conversation ID) to operate on +-Z, --zinvite ZINVITE ZINVITE code (short code) of conversation to operate on +``` + +### Export-Specific Options + +The `export` command has many additional options: + +```txt +-z, --zid ZID ZID on which to do an export +-Z, --zinvite ZINVITE ZINVITE code on which to perform an export +-X, --include-xid Include user xids in output +-u, --user-id USER_ID Export all conversations associated with USER_ID, and place in zip file +-f, --filename FILENAME Name of output file (should be zip for csv out) +-t, --at-time AT_TIME A string of YYYY-MM-DD-HH-MM-SS (in UTC) or ms-timestamp since epoch +-T, --at-times AT_TIMES A vector of strings of --at-time format +-F, --format FORMAT Either csv, excel or (soon) json +-M, --update-math Update math +-P, --update-postgres Update postgres +-h, --help Print help and exit +``` + +## Running Commands from the REPL + +When working with the REPL, you can run and manage systems directly using functions in the `polismath.runner` namespace. This approach provides more flexibility and interactive development capabilities. + +### Starting and Stopping Systems + +```clojure +;; First require the runner namespace and system definitions +(require '[polismath.runner :as runner] + '[polismath.system :as system]) + +;; Run the full system (poller + tasks) +(runner/run! system/full-system) + +;; Run with configuration overrides +(runner/run! system/full-system {:math-env :preprod + :poll-from-days-ago 0.1}) + +;; Run just the poller system +(runner/run! system/poller-system) + +;; Run the export system +(runner/run! system/export-system) + +;; Stop any running system +(runner/stop!) + +;; Reset the system (stops, reloads code, and restarts) +(runner/system-reset!) +``` + +### Working with Conversations + +```clojure +;; Load a conversation by ID +(def conv (conv-man/load-or-init (:conversation-manager runner/system) 12345)) + +;; Update a conversation +(def updated-conv (conv/conv-update conv [])) + +;; Queue updates for a conversation +(conv-man/queue-message-batch! (:conversation-manager runner/system) + :votes + 12345 + []) +``` + +### Running Tests in the REPL + +```clojure +;; Run the test suite directly +(require '[test-runner]) +(test-runner/-main) + +;; Run a specific test namespace +(require '[clojure.test :as test]) +(test/run-tests 'conversation-test) +``` + +### Lifecycle Management + +The REPL approach lets you start, modify, and restart components of the system without restarting your entire application: + +```clojure +;; Initialize a system but don't start it +(runner/init! system/base-system) + +;; Start the system +(runner/start!) + +;; Stop the system +(runner/stop!) + +;; Make code changes... + +;; Reload code and restart +(require '[clojure.tools.namespace.repl :as namespace.repl]) +(namespace.repl/refresh) +(runner/start!) + +;; Or use the combined reset function +(runner/system-reset!) +``` + +## Examples + +### Running the Full System + +To run both the poller and tasks systems: + +```sh +clojure -M:run full +``` + +### Updating a Specific Conversation + +Update a conversation by ZID (numeric ID): + +```sh +clojure -M:run update -z 12345 +``` + +Or by ZINVITE (short code): + +```sh +clojure -M:run update -Z abc123 +``` + +### Running with Docker Compose + +With Docker Compose, you can run commands within the math container: + +```sh +docker compose run math clojure -M:run +``` + +### Exporting Data + +Export a conversation to a zip file: + +```sh +clojure -M:run export -z 12345 -f export.zip +``` + +Export all conversations for a user: + +```sh +clojure -M:run export -u 67890 -f user-exports.zip +``` + +Export a conversation at a specific point in time: + +```sh +clojure -M:run export -Z abc123 -t 2023-01-15-12-00-00 -f historical.zip +``` + +### System Timeout and Restart + +The `bin/run` script uses a timeout mechanism to automatically restart the system every 4 hours: + +```sh +timeout -s KILL 14400 clojure -M:run full +``` + +This runs the full system for up to 4 hours (14400 seconds), then kills it and restarts. This prevents memory leaks and ensures the system stays healthy during long-running operations. + +## Getting Help + +To see available options for any command: + +```sh +clojure -M:run --help +``` + +For export-specific help: + +```sh +clojure -M:run export --help +``` + +## Running Tests + +To run the test suite: + +```sh +clojure -M:test +``` + +Or from within a REPL: + +```clojure +(require '[test-runner]) +(test-runner/-main) +``` diff --git a/math/doc/configuration.md b/math/doc/configuration.md index f9fe9958ab..0140812356 100644 --- a/math/doc/configuration.md +++ b/math/doc/configuration.md @@ -1,28 +1,25 @@ +# Polismath Configuration Guide -# Math worker configuration +This document provides detailed information about configuring the Polismath service, the mathematical backend for the Polis system. -This set of documentation is currently incomplete, but describes a couple of the more important bits of configuration in the system. +## Configuration Methods -## Environment variables +The Polismath service can be configured in several ways: -There are a number of variables for tuning and tweaking the system, many of which are exposed via environment variables. -Please see [`src/polismath/components/config.clj`](https://github.com/pol-is/polisMath/blob/master/src/polismath/components/config.clj#L51) for the complete listing of environment variables. +1. **Environment Variables** (primary method) +2. **Configuration File** (resources/config.edn - secondary) +3. **Runtime Configuration** (via REPL) -The ones you're most frequently to need to tweak for one reason or another: +Configuration values are merged in the following order of precedence (highest to lowest): -* **`MATH_ENV`**: This defaults to `dev`, for local development environments. - Traditionally we've set this to `prod` and `preprod` for our production and pre-production deployments specifically. - This value is used in keying the math export json blobs as found in the `math_main` and other tables in the database. - This makes it possible to run multiple math environments (dev, testing, prod, preprod) all on the same database of votes. - This setting is something of a relic from an old architecture where prod and preprod environments ran off of the same database, and with the docker infrastructure is generally no longer needed. - Nevertheless, when you start the math server, you will need to run it with the same **`MATH_ENV`** setting as you ran the math worker with. -* **`POLL_FROM_DAYS_AGO`**: This defaults to 10 (at the time of this writing). - Conversations which have had vote or moderation activity in the specified range will be loaded into memory, and will be updated. - This prevents old inactive conversations from being loaded into memory every time the poller starts. +1. Runtime overrides +2. Environment variables +3. Configuration file +4. Default values -You'll also need to pass database credentials. If using docker compose, this will be inherited from the `.env` file or process environment in which docker is being run. +## Required Configuration - **`DATABASE_URL`**: url for the database: +At minimum, you need to provide: `postgres://:@:/` @@ -46,3 +43,162 @@ The connection pool automatically: ### Connection Pool Monitoring The service logs connection pool configuration at startup and provides health check functionality. Connection failures are automatically retried up to 3 times with exponential backoff. +- **Database Connection**: The database URL for connecting to PostgreSQL + +## Environment Variables + +Environment variables are the primary method for configuring the system. When using Docker Compose, these can be set in a `.env` file at the project root. + +### Database Configuration + +| Variable | Description | Default | +|----------|-------------|---------| +| `DATABASE_URL` | PostgreSQL connection string in the format `postgres://username:password@hostname:port/database` | None (Required) | +| `DATABASE_POOL_SIZE` | Connection pool size | 3 | +| `DATABASE_IGNORE_SSL` | Whether to ignore SSL for database connections | false | +| `DATABASE_FOR_READS_NAME` | Database name for read operations if different | None | + +### Environment Configuration + +| Variable | Description | Default | +|----------|-------------|---------| +| `MATH_ENV` | Environment setting (`dev`, `prod`, `preprod`) | `dev` | +| `LOGGING_LEVEL` | Logging level (`trace`, `debug`, `info`, `warn`, `error`, `fatal`, `report`) | `warn` | +| `POLL_FROM_DAYS_AGO` | How far back to poll for conversation updates (in days) | 10 | + +> **Note:** When using Docker Compose, use `MATH_LOG_LEVEL` instead of `LOGGING_LEVEL`. Docker Compose translates this to `LOGGING_LEVEL` internally. + +### Polling Configuration + +| Variable | Description | Default | +|----------|-------------|---------| +| `VOTE_POLLING_INTERVAL` | Interval for polling votes (milliseconds) | 1000 | +| `MOD_POLLING_INTERVAL` | Interval for polling moderation actions (milliseconds) | 1000 | +| `MATH_ZID_BLOCKLIST` | Comma-separated list of conversation IDs to block from processing | None | +| `MATH_ZID_ALLOWLIST` | Comma-separated list of conversation IDs to exclusively process | None | + +### Math Processing Configuration + +| Variable | Description | Default | +|----------|-------------|---------| +| `MATH_MATRIX_IMPLEMENTATION` | Matrix implementation to use (`vectorz` recommended) | `vectorz` | +| `MATH_CUTOFF_MEDIUM` | Maximum size before running in medium mode | None | +| `MATH_CUTOFF_LARGE` | Maximum size before running in large mode | None | +| `MATH_CUTOFF_MAX_PTPTS` | Maximum participants before rejecting new participants | None | +| `MATH_CUTOFF_MAX_CMNTS` | Maximum comments before rejecting new comments | None | +| `RECOMPUTE` | Whether to recompute conversations from scratch | false | + +### Export Configuration + +| Variable | Description | Default | +|----------|-------------|---------| +| `EXPORT_EXPIRY_DAYS` | Days before exported data expires | 6 | +| `EXPORT_SERVER_AUTH_USERNAME` | Username for export server authentication | None | +| `EXPORT_SERVER_AUTH_PASS` | Password for export server authentication | None | + +### AWS Configuration (for exports) + +| Variable | Description | Default | +|----------|-------------|---------| +| `AWS_ACCESS_KEY` | AWS access key ID | None | +| `AWS_SECRET_KEY` | AWS secret access key | None | + +### API Authentication + +| Variable | Description | Default | +|----------|-------------|---------| +| `WEBSERVER_USERNAME` | Username for webserver authentication | None | +| `WEBSERVER_PASS` | Password for webserver authentication | None | + +## Setting Environment Variables + +### Docker Environment + +With Docker Compose, you can use a `.env` file: + +```sh +DATABASE_URL=postgres://username:password@hostname:port/database +MATH_ENV=dev +MATH_LOG_LEVEL=info # Docker Compose translates this to LOGGING_LEVEL +``` + +Then run: + +```sh +docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile postgres up +``` + +### Direct Environment + +When running without Docker, set environment variables directly: + +```bash +export DATABASE_URL=postgres://username:password@hostname:port/database +export MATH_ENV=dev +export LOGGING_LEVEL=info # Use LOGGING_LEVEL directly when not using Docker Compose +clojure -M:dev +``` + +## Runtime Configuration + +When using the REPL, you can override configuration when starting the system: + +```clojure +(require '[polismath.runner :as runner] + '[polismath.system :as system]) + +;; Run with custom configuration +(runner/run! system/full-system + {:math-env :preprod + :poll-from-days-ago 0.1}) +``` + +## Logging Configuration + +The system uses Timbre for logging. You can configure it in several ways: + +1. Set the environment variable: + - When running directly: use `LOGGING_LEVEL` + - When using Docker Compose: use `MATH_LOG_LEVEL` (Docker translates this to `LOGGING_LEVEL` internally) + +2. Configure at runtime: + + ```clojure + (require '[taoensso.timbre :as log]) + (log/set-level! :debug) ;; Set to debug level + ``` + +Available log levels (from least to most verbose): + +- `:report` +- `:fatal` +- `:error` +- `:warn` (default) +- `:info` +- `:debug` +- `:trace` + +You can also configure logging to output to a file by setting the `LOGGING_FILE` environment variable. + +## Advanced Configuration + +For more advanced configuration options, see the `rules` map in [src/polismath/components/config.clj](../src/polismath/components/config.clj). + +## System Components + +The Polismath system consists of several components that can be configured and started independently: + +- **Base System**: Core functionality without polling +- **Poller System**: Base system plus database polling for votes and moderation +- **Task System**: Handles auxiliary tasks like exports +- **Full System**: Combines poller and task systems + +To run a specific subsystem, use: + +```clojure +;; In REPL +(runner/run! system/poller-system {:math-env :dev}) + +;; Or via command line +clojure -M:run poller +``` diff --git a/math/doc/update.md b/math/doc/update.md new file mode 100644 index 0000000000..1876d5663c --- /dev/null +++ b/math/doc/update.md @@ -0,0 +1,153 @@ +# Conversation Update - Analysis and Recommendations + +_Generated by gemini-2.0-pro-exp on 2025-03-15_ + +Okay, I've reviewed the provided code, focusing on the `conv-update` functionality and related areas across the specified files. Here's a breakdown of potential issues, inefficiencies, and areas for improvement, keeping in mind the concerns about scalability (O(log n), O(c^n) problems). + +**Overall Structure and Key Functions** + +The conversation update process is primarily driven by these functions: + +* **`user.clj`:** This seems to be a general utility and REPL interaction namespace. It includes functions for loading conversations (`load-conv`, `get-conv`), plotting, and some data analysis. It's the entry point for many manual operations. +* **`polismath.runner/run!`:** The main entry point for starting the system. It initializes and starts components. +* **`polismath.conv_man/queue-message-batch!`:** This function queues messages (votes, moderation actions, report generation requests) for processing by a conversation actor. It's the main way the outside world interacts with a conversation. +* **`polismath.conv_man/conv-actor`:** Creates a core.async go-loop that processes messages for a _single_ conversation. It maintains the conversation state in an atom (`:conv`). +* **`polismath.conv_man/react-to-messages`:** A multimethod that dispatches based on the message type (`:votes`, `:moderation`, `:generate_report_data`). This is where the actual update logic is selected. +* **`polismath.conv_man/conv-update`:** This function _calls_ the core math update function (`polismath.math.conversation/conv-update`), handles profiling, validation, and error handling, and then triggers database writes. It's a crucial wrapper. +* **`polismath.math.conversation/conv-update`:** This is the _heart_ of the update process. It uses `plumbing.graph` to define a dataflow graph that computes the new conversation state. It dispatches to either `small-conv-update` or `large-conv-update` based on the number of participants. +* **`polismath.math.conversation/small-conv-update-graph` and `large-conv-update-graph`:** These are the Plumbing graph definitions. They specify the data dependencies and calculations needed to update the conversation. The large version uses a mini-batch PCA for scalability. +* **`polismath.conv_man/write-conv-updates!`:** Writes the updated conversation data to the database (Postgres). This is done in a separate thread (`async/thread`) to avoid blocking the main update process. +* **`polismath.meta.microscope/recompute`:** A function to trigger a full recomputation of a conversation, likely used for debugging or recovery. + +**Potential Issues and Areas for Improvement** + +1. **`polismath.math.conversation/conv-update` Dispatch Logic:** + + * The choice between `small-conv-update` and `large-conv-update` is based solely on the number of participants (`n-ptpts`). It uses a hardcoded `large-cutoff` (defaulting to 10000). This might not be optimal. Consider: + * **Adaptive Cutoff:** The cutoff should perhaps be dynamic, based on available memory, system load, or other factors. + * **Comment Count:** The number of _comments_ (`n-cmts`) also significantly impacts performance. The dispatch logic should consider both participants and comments. A conversation with few participants but many comments might still benefit from the large-conv-update approach. + * **Combined Metric:** Perhaps a combined metric (e.g., `n-ptpts * n-cmts` or `n-ptpts * log(n-cmts)`) would be a better indicator of when to switch to the large-conv-update. + +2. **`plumbing.graph` Complexity:** + + * The `small-conv-update-graph` and `large-conv-update-graph` are complex. While `plumbing.graph` helps manage dependencies, it's crucial to analyze the computational complexity of _each step_ within the graph. + * **`pca/wrapped-pca`:** This is likely a performance bottleneck. The `large-conv-update` uses mini-batch PCA to mitigate this, but the `small-conv-update` uses the full PCA. For conversations that are _just_ below the `large-cutoff`, this could be slow. Consider always using mini-batch PCA, perhaps with a smaller batch size for smaller conversations. + * **`clusters/kmeans`:** K-means clustering is used extensively (for base clusters, group clusters, and subgroup clusters). The complexity of K-means can vary, but it's generally considered O(n*k*i*d) where n is the number of data points, k is the number of clusters, i is the number of iterations, and d is the number of dimensions. + * **Multiple `kmeans` Calls:** The code performs K-means clustering _multiple times_ with different `k` values (for finding the optimal number of clusters). This is computationally expensive. The `group-clusterings` and `subgroup-clusterings` calculations are particularly concerning. + * **Silhouette Calculation:** The silhouette calculation (`clusters/silhouette`) is also expensive, as it involves calculating distances between all pairs of points within a cluster. This is done for _every_ `k` value. + * **Optimization Strategies:** + * **Approximate K-means:** Explore approximate K-means algorithms (e.g., mini-batch K-means, BIRCH) to reduce the computational cost, especially for the base clusters. + * **Reduce `k` Range:** Limit the range of `k` values considered, especially for subgroup clusters. Do you really need to test every `k` from 2 to `max-k`? Could you use a heuristic to narrow the search? + * **Cache Distances:** The `bucket-dists` calculation (which computes distances between base clusters) is used repeatedly. Ensure this result is cached and reused as much as possible. + * **Subgroup Cluster Heuristics:** The subgroup clustering logic is complex. Consider if you _really_ need two levels of clustering. If so, explore more efficient hierarchical clustering algorithms. The current approach of running K-means for every group and every possible `k` is likely a major scalability issue. + * **`repness/conv-repness` and `repness/participant-stats`:** These functions calculate representativeness scores. Examine their implementation for potential inefficiencies. Matrix operations within these functions could be bottlenecks. + * **`group-votes` and `subgroup-votes`:** These functions aggregate votes by group and subgroup. The nested loops and use of `get-in` could be optimized. Consider using transducers or specialized data structures for faster aggregation. + +3. **Memory Usage and Leaks:** + + * **`conv` Atom:** The entire conversation state is held in an atom (`:conv` within the `conv-actor`). For very large conversations, this could lead to high memory usage. + * **`raw-rating-mat`:** This matrix stores _all_ votes. For long-running conversations, this matrix can become extremely large. + * **Sparsity:** The `raw-rating-mat` is likely very sparse (most participants haven't voted on most comments). Ensure that the `named-matrix` implementation efficiently handles sparse matrices. If not, consider using a sparse matrix library. + * **Compaction/Archiving:** Implement a mechanism to periodically compact or archive old votes. For example, you could keep only the most recent N votes per participant or move older votes to a separate, less frequently accessed data store. + * **`profile-data`:** The code accumulates profiling data in an atom (`:profile-data`). While this is useful for debugging, it could grow unbounded. Ensure that this data is periodically flushed or limited in size. + * **Closure over Large Data:** Be cautious about closures that might inadvertently capture large data structures and prevent garbage collection. This is a common source of memory leaks in Clojure. The use of `plmb/map-from-keys` and similar functions should be reviewed carefully. + +4. **Database Interactions (`write-conv-updates!`):** + + * **Multiple `async/thread` Calls:** The `write-conv-updates!` function uses `async/thread` for each database write (main, bidtopid, ptptstats). While this provides concurrency, it could lead to excessive thread creation for large conversations. Consider using a fixed-size thread pool to limit the number of concurrent database operations. + * **Transaction Management:** The code doesn't explicitly use database transactions. If one of the database writes fails, the system could end up in an inconsistent state. Wrap the database writes in a transaction to ensure atomicity. + * **Bulk Inserts:** If possible, use bulk insert operations to reduce the overhead of individual database writes. + +5. **Error Handling:** + + * **`handle-errors`:** The error handling is reasonable (logging, notification, re-queueing messages). However, consider: + * **Retry Limits:** The code re-queues failed messages indefinitely. This could lead to an infinite loop if a message consistently causes an error. Implement a retry limit or a dead-letter queue. + * **Error Reporting:** The error reporting could be more informative. Include details about the specific error, the message being processed, and the conversation state. + +6. **Debugging and Observability:** + + * **Logging:** The logging is generally good, but consider adding more context to log messages (e.g., the current `math-tick`, the number of votes processed). + * **Metrics:** The code sends some basic metrics (e.g., `math.pca.compute.fail`). Expand the metrics to include: + * Conversation size (participants, comments, votes) + * Update time (for different stages of the update process) + * Queue sizes (message-chan, retry-chan) + * Database write times + * K-means iteration counts + * Silhouette scores + * **Tracing:** Use a tracing library (e.g., `tracing-utils`) to get more detailed information about the execution flow and timing of individual functions. + * **Visualization:** The `user.clj` file includes functions for plotting conversation data. These visualizations are crucial for understanding conversation dynamics and identifying potential issues. Expand these visualizations to include: + * Histograms of vote distributions + * Plots of K-means convergence + * Visualizations of the `raw-rating-mat` (e.g., a heatmap) + +7. **`conv-man/load-or-init`:** + * The function reads the entire conversation history from the database when loading a conversation. For large conversations, this is inefficient. Consider loading only a summary or a recent snapshot of the conversation state. + * The function associates `:recompute :reboot` when loading an existing conversation. This seems unnecessary and potentially confusing. + +8. **`user.clj` plotting functions:** + * The plotting functions in `user.clj` are useful for interactive analysis, but they could be made more robust and efficient. Consider using a dedicated plotting library (e.g., Oz, which is already a dependency) for more advanced visualizations. + +**Specific Code Snippets and Concerns** + +* **`polismath.math.conversation/agg-bucket-votes-for-tid`:** + + ```clojure + (mapv ; for each bucket + (fn [pids] + (->> pids + ; get votes for the tid from each ptpt in group + (map (fn [pid] (get (get person-rows (pid-to-row pid)) idx))) + ; filter votes you don't want to count + (filter filter-cond) + ; count + (count))) + bid-to-pid) + ``` + + This function iterates over `bid-to-pid`, which represents base clusters. Inside, it iterates over `pids` (members of a bucket) and uses `get` multiple times to access the vote matrix. This nested iteration and repeated `get` calls could be inefficient. Consider using matrix/vector operations for faster access. + +* **`polismath.math.conversation/group-votes`:** + + ```clojure + :votes (plmb/map-from-keys + (fn [tid] + {:A (count-fn tid :A) + :D (count-fn tid :D) + :S (count-fn tid :S)}) + (keys votes-base)) + ``` + + This uses `plmb/map-from-keys` which might be less efficient than direct iteration and aggregation. The `count-fn` is called multiple times for each `tid`. + +* **`polismath.math.conversation/priority-metric`**: + + ```clojure + (matrix/pow + (if is-meta + meta-priority + (* (importance-metric A P S E) + ;; scale by a factor which lets new comments bubble up + (+ 1 (* 8 (matrix/pow 2 (/ S -5)))))) + 2)) + ``` + + The use of `matrix/pow` here, even with a small exponent (2), might be more computationally expensive than simple multiplication. Consider replacing `(matrix/pow x 2)` with `(* x x)`. + +* **`take-all!` and `take-all!!`:** These macros/functions are generally fine, but be aware of the potential for blocking if the channel is constantly being filled. + +**Recommendations Summary** + +1. **Optimize `kmeans` and Silhouette:** This is likely the biggest performance bottleneck. Explore approximate K-means, reduce the range of `k` values, and cache distances. +2. **Sparse Matrix Representation:** Ensure `raw-rating-mat` uses a sparse matrix implementation. +3. **Compact/Archive Votes:** Limit the size of `raw-rating-mat` by archiving or compacting old votes. +4. **Adaptive `conv-update` Dispatch:** Choose between `small-conv-update` and `large-conv-update` based on a combined metric of participants and comments. Consider always using mini-batch PCA. +5. **Thread Pool for DB Writes:** Use a fixed-size thread pool for database operations. +6. **Database Transactions:** Wrap database writes in transactions. +7. **Bulk Inserts:** Use bulk insert operations where possible. +8. **Retry Limits:** Implement retry limits or a dead-letter queue for failed messages. +9. **Enhanced Metrics and Logging:** Add more detailed metrics and logging. +10. **Load Conversation Summaries:** Load only necessary data in `load-or-init`. +11. **Review Closures:** Carefully review closures for unintended capture of large data structures. +12. **Optimize `group-votes` and related functions:** Use transducers or more efficient data structures for aggregation. + +By addressing these points, you can significantly improve the scalability and robustness of the conversation update process. The key is to focus on reducing the computational complexity of the core algorithms (PCA, K-means), minimizing memory usage, and optimizing database interactions. diff --git a/math/src/polismath/meta/metrics.clj b/math/src/polismath/meta/metrics.clj index b61be65b49..12799871df 100644 --- a/math/src/polismath/meta/metrics.clj +++ b/math/src/polismath/meta/metrics.clj @@ -45,11 +45,12 @@ (defn- send-metric-values [metric-sender values] (let [{:keys [api-key hostname remote-port]} (get-config metric-sender)] - (log/info "sending metric data " values " to " hostname ":" remote-port) - (send-data metric-sender - (make-send-string (-> metric-sender :config :math-env) - api-key - values)))) + (when (and api-key hostname) ; Only send metrics if properly configured + (log/info "sending metric data " values " to " hostname ":" remote-port) + (send-data metric-sender + (make-send-string (-> metric-sender :config :math-env) + api-key + values))))) ;; ## Public API From 8ce4e99328187737fb7328447c5097b04051715f Mon Sep 17 00:00:00 2001 From: Bennie Rosas Date: Sat, 25 Oct 2025 02:47:29 -0500 Subject: [PATCH 2/7] remove MATH_ENV, WEBSERVER, and other dead code from server --- server/app.ts | 67 ------------- server/example.env | 1 - server/src/config.ts | 3 - server/src/email/senders.ts | 40 +------- server/src/routes/dataExport.ts | 49 --------- server/src/routes/metadata.ts | 4 +- server/src/routes/notify.ts | 36 ------- server/src/server.ts | 172 +------------------------------- server/src/utils/common.ts | 53 ---------- 9 files changed, 4 insertions(+), 421 deletions(-) delete mode 100644 server/src/routes/dataExport.ts diff --git a/server/app.ts b/server/app.ts index e834ea585c..4bd88ff9ba 100644 --- a/server/app.ts +++ b/server/app.ts @@ -132,10 +132,6 @@ import { handle_POST_query_participants_by_metadata, handle_PUT_participants_extended, } from "./src/routes/participation"; -import { - handle_GET_dataExport, - handle_GET_dataExport_results, -} from "./src/routes/dataExport"; import { handle_GET_votes_famous, handle_GET_votes_me, @@ -152,7 +148,6 @@ import { handle_GET_notifications_subscribe, handle_GET_notifications_unsubscribe, handle_POST_convSubscriptions, - handle_POST_notifyTeam, } from "./src/routes/notify"; import { handle_GET_reports, @@ -271,10 +266,8 @@ helpersInitialized.then( handle_GET_zinvites, handle_POST_contexts, - handle_POST_contributors, handle_POST_metrics, handle_POST_sendCreatedLinkToEmail, - handle_POST_sendEmailExportReady, handle_POST_tutorial, handle_POST_zinvites, } = o; @@ -358,21 +351,6 @@ helpersInitialized.then( handle_GET_math_correlationMatrix ); - app.get( - "/api/v3/dataExport", - moveToBody, - hybridAuth(assignToP), - need( - "conversation_id", - getConversationIdFetchZid, - assignToPCustom("zid") - ), - need("conversation_id", getStringLimitLength(1, 1000), assignToP), - want("format", getStringLimitLength(1, 100), assignToP), - want("unixTimestamp", getStringLimitLength(99), assignToP), - handle_GET_dataExport - ); - app.get( "/api/v3/reportExport/:report_id/:report_type", moveToBody, @@ -382,20 +360,6 @@ helpersInitialized.then( handle_GET_reportExport ); - app.get( - "/api/v3/dataExport/results", - moveToBody, - hybridAuth(assignToP), - need( - "conversation_id", - getConversationIdFetchZid, - assignToPCustom("zid") - ), - need("conversation_id", getStringLimitLength(1, 1000), assignToP), - want("filename", getStringLimitLength(1, 1000), assignToP), - handle_GET_dataExport_results - ); - // TODO doesn't scale, stop sending entire mapping. app.get( "/api/v3/bidToPid", @@ -576,26 +540,6 @@ helpersInitialized.then( app.get("/perfStats_9182738127", moveToBody, handle_GET_perfStats); - app.post( - "/api/v3/sendEmailExportReady", - need("webserver_username", getStringLimitLength(1, 999), assignToP), - need("webserver_pass", getStringLimitLength(1, 999), assignToP), - need("email", getEmail, assignToP), - // we actually need conversation_id to build a url - need("conversation_id", getStringLimitLength(1, 1000), assignToP), - need("filename", getStringLimitLength(9999), assignToP), - handle_POST_sendEmailExportReady - ); - - app.post( - "/api/v3/notifyTeam", - need("webserver_username", getStringLimitLength(1, 999), assignToP), - need("webserver_pass", getStringLimitLength(1, 999), assignToP), - need("subject", getStringLimitLength(9999), assignToP), - need("body", getStringLimitLength(99999), assignToP), - handle_POST_notifyTeam - ); - app.get( "/api/v3/domainWhitelist", moveToBody, @@ -1937,17 +1881,6 @@ helpersInitialized.then( handle_GET_verification ); - app.post( - "/api/v3/contributors", - hybridAuthOptional(assignToP), - need("agreement_version", getIntInRange(1, 999999), assignToP), - need("name", getStringLimitLength(746), assignToP), - need("email", getStringLimitLength(256), assignToP), - need("github_id", getStringLimitLength(256), assignToP), - need("company_name", getStringLimitLength(746), assignToP), - handle_POST_contributors - ); - app.post( "/api/v3/metrics", hybridAuthOptional(assignToP), diff --git a/server/example.env b/server/example.env index 3acd8d2240..50fa431e88 100644 --- a/server/example.env +++ b/server/example.env @@ -10,7 +10,6 @@ API_SERVER_PORT=5000 DATABASE_URL=postgres://postgres:PdwPNS2mDN73Vfbc@localhost:5432/polis-test # TEST DEV_MODE=true DOMAIN_OVERRIDE= -MATH_ENV=dev POLIS_FROM_ADDRESS="Example " SERVER_LOG_LEVEL=info SES_ENDPOINT=http://localhost:8005 diff --git a/server/src/config.ts b/server/src/config.ts index 5c435d57f8..ac48694439 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -113,7 +113,6 @@ export default { process.env.MAX_REPORT_CACHE_DURATION || "3600000", 10 ), - mathEnv: process.env.MATH_ENV as string, nodeEnv: process.env.NODE_ENV as string, isTesting: isTrue(process.env.TESTING), openaiApiKey: process.env.OPENAI_API_KEY || null, @@ -169,8 +168,6 @@ export default { // Deprecated encryptionPassword: process.env.ENCRYPTION_PASSWORD_00001 as string, - webserverPass: process.env.WEBSERVER_PASS as string, - webserverUsername: process.env.WEBSERVER_USERNAME as string, }; // Use this function when a value should default to true if not set. diff --git a/server/src/email/senders.ts b/server/src/email/senders.ts index 8bfc711540..df06e96c30 100644 --- a/server/src/email/senders.ts +++ b/server/src/email/senders.ts @@ -90,42 +90,4 @@ async function sendMultipleTextEmails( return results; } -async function emailTeam(subject: string, body: string): Promise { - let adminEmails: string[] = []; - try { - if (Config.adminEmails) { - adminEmails = JSON.parse(Config.adminEmails); - } - } catch (err) { - logger.error("polis_err_email_config_parse_failure", { - message: "Failed to parse JSON from Config.adminEmails.", - configValue: Config.adminEmails, - timestamp: new Date().toISOString(), - cause: { - name: (err as Error).name, - message: (err as Error).message, - stack: (err as Error).stack, - }, - }); - return; - } - - if (!Config.polisFromAddress) { - logger.error("polis_err_email_config_missing_sender", { - message: "The 'polisFromAddress' is not configured.", - timestamp: new Date().toISOString(), - }); - return; - } - - if (adminEmails.length > 0) { - await sendMultipleTextEmails( - Config.polisFromAddress, - adminEmails, - subject, - body - ); - } -} - -export { sendMultipleTextEmails, sendTextEmail, emailTeam }; +export { sendMultipleTextEmails, sendTextEmail }; diff --git a/server/src/routes/dataExport.ts b/server/src/routes/dataExport.ts deleted file mode 100644 index 89f6e5a221..0000000000 --- a/server/src/routes/dataExport.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { getUserInfoForUid2 } from "../user"; -import { doAddDataExportTask } from "../utils/common"; -import Config from "../config"; -import { failJson } from "../utils/fail"; -import AWS from "aws-sdk"; -import { UserInfo } from "../d"; - -AWS.config.update({ region: Config.awsRegion }); -const s3Client = new AWS.S3({ apiVersion: "2006-03-01" }); - -function handle_GET_dataExport( - req: { p: { uid?: number; zid: number; unixTimestamp: number; format: any } }, - res: { json: (arg0: {}) => void } -) { - getUserInfoForUid2(req.p.uid) - .then((user: UserInfo) => { - return doAddDataExportTask( - Config.mathEnv, - user.email!, - req.p.zid, - req.p.unixTimestamp * 1000, - req.p.format, - Math.abs((Math.random() * 999999999999) >> 0) - ) - .then(() => { - res.json({}); - }) - .catch((err: any) => { - failJson(res, 500, "polis_err_data_export123", err); - }); - }) - .catch((err: any) => { - failJson(res, 500, "polis_err_data_export123b", err); - }); -} - -function handle_GET_dataExport_results( - req: { p: { filename: string } }, - res: { redirect: (arg0: any) => void } -) { - const url = s3Client.getSignedUrl("getObject", { - Bucket: "polis-datadump", - Key: Config.mathEnv + "/" + req.p.filename, - Expires: 60 * 60 * 24 * 7, - }); - res.redirect(url); -} - -export { handle_GET_dataExport, handle_GET_dataExport_results }; diff --git a/server/src/routes/metadata.ts b/server/src/routes/metadata.ts index af491e98a6..fc91ebf5e4 100644 --- a/server/src/routes/metadata.ts +++ b/server/src/routes/metadata.ts @@ -6,7 +6,7 @@ import { finishArray, finishOne } from "../server-helpers"; import { sql_participant_metadata_answers } from "../db/sql"; import logger from "../utils/logger"; import pg from "../db/pg-query"; -import Utils, { isConversationOwner } from "../utils/common"; +import { isConversationOwner } from "../utils/common"; function getZidForAnswer( pmaid: any, @@ -66,7 +66,7 @@ function handle_DELETE_metadata_answers( ); return; } - Utils.isConversationOwner(zid, uid, function (err: any) { + isConversationOwner(zid, uid, function (err: any) { if (err) { failJson( res, diff --git a/server/src/routes/notify.ts b/server/src/routes/notify.ts index 52a03925a6..1a92644e0f 100644 --- a/server/src/routes/notify.ts +++ b/server/src/routes/notify.ts @@ -3,7 +3,6 @@ import crypto from "crypto"; import { encode } from "html-entities"; import { Promise as BluebirdPromise } from "bluebird"; -import { emailTeam } from "../email/senders"; import { failJson } from "../utils/fail"; import { getConversationInfo } from "../conversation"; import { getNumberOfCommentsRemaining } from "../comment"; @@ -498,43 +497,8 @@ function handle_POST_convSubscriptions( } } -function handle_POST_notifyTeam( - req: { - p: { - webserver_pass: string | undefined; - webserver_username: string | undefined; - subject: any; - body: any; - }; - }, - res: { - status: (arg0: number) => { - (): any; - new (): any; - json: { (arg0: {}): void; new (): any }; - }; - } -) { - if ( - req.p.webserver_pass !== Config.webserverPass || - req.p.webserver_username !== Config.webserverUsername - ) { - return failJson(res, 403, "polis_err_notifyTeam_auth"); - } - const subject = req.p.subject; - const body = req.p.body; - emailTeam(subject, body) - .then(() => { - res.status(200).json({}); - }) - .catch((err: any) => { - return failJson(res, 500, "polis_err_notifyTeam", err); - }); -} - export { handle_GET_notifications_subscribe, handle_GET_notifications_unsubscribe, handle_POST_convSubscriptions, - handle_POST_notifyTeam, }; diff --git a/server/src/server.ts b/server/src/server.ts index e62515ffb4..5a23d5fa12 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -23,11 +23,7 @@ import { } from "./d"; import { detectLanguage } from "./comment"; import logger from "./utils/logger"; -import { - emailTeam, - sendMultipleTextEmails, - sendTextEmail, -} from "./email/senders"; +import { sendMultipleTextEmails, sendTextEmail } from "./email/senders"; AWS.config.update({ region: Config.awsRegion }); const devMode = Config.isDevMode; @@ -136,71 +132,6 @@ function initializePolisHelpers() { return next(); } - function doAddDataExportTask( - math_env: string | undefined, - email: string, - zid: number, - atDate: number, - format: string, - task_bucket: number - ) { - return pg.queryP( - "insert into worker_tasks (math_env, task_data, task_type, task_bucket) values ($1, $2, 'generate_export_data', $3);", - [ - math_env, - { - email: email, - zid: zid, - "at-date": atDate, - format: format, - }, - task_bucket, // TODO hash the params to get a consistent number? - ] - ); - } - if ( - Config.runPeriodicExportTests && - !devMode && - Config.mathEnv === "preprod" - ) { - const runExportTest = () => { - const math_env = "prod"; - const email = Config.adminEmailDataExportTest; - const zid = 12480; - const atDate = Date.now(); - const format = "csv"; - const task_bucket = Math.abs((Math.random() * 999999999999) >> 0); - doAddDataExportTask( - math_env, - email, - zid, - atDate, - format, - task_bucket - ).then(() => { - setTimeout(() => { - pg.queryP( - "select * from worker_tasks where task_type = 'generate_export_data' and task_bucket = ($1);", - [task_bucket] - ).then((rows: string | any[]) => { - const ok = rows && rows.length; - let newOk; - if (ok) { - newOk = rows[0].finished_time > 0; - } - if (ok && newOk) { - logger.info("runExportTest success"); - } else { - logger.error("runExportTest failed"); - emailBadProblemTime("Math export didn't finish."); - } - }); - }, 10 * 60 * 1000); // wait 10 minutes before verifying - }); - }; - setInterval(runExportTest, 6 * 60 * 60 * 1000); // every 6 hours - } - const getServerNameWithProtocol = Config.getServerNameWithProtocol; function handle_POST_metrics( @@ -391,14 +322,6 @@ ${message}`; }); } - function emailBadProblemTime(message: string) { - const body = `Yo, there was a serious problem. Here's the message: - -${message}`; - - return emailTeam("Polis Bad Problems!!!", body); - } - function handle_GET_verification( req: { p: { e: any } }, res: { @@ -603,56 +526,6 @@ Email verified! You can close this tab or hit the back button. ); } - function handle_POST_sendEmailExportReady( - req: { - p: { - webserver_pass: string | undefined; - webserver_username: string | undefined; - email: any; - conversation_id: string; - filename: any; - }; - }, - res: { - status: (arg0: number) => { - (): any; - new (): any; - json: { (arg0: {}): void; new (): any }; - }; - } - ) { - if ( - req.p.webserver_pass !== Config.webserverPass || - req.p.webserver_username !== Config.webserverUsername - ) { - return failJson(res, 403, "polis_err_sending_export_link_to_email_auth"); - } - - const serverUrl = Config.getServerUrl(); - const email = req.p.email; - const subject = - "Polis data export for conversation pol.is/" + req.p.conversation_id; - const fromAddress = `Polis Team <${Config.adminEmailDataExport}>`; - const body = `Greetings - -You created a data export for conversation ${serverUrl}/${req.p.conversation_id} that has just completed. You can download the results for this conversation at the following url: - -${serverUrl}/api/v3/dataExport/results?filename=${req.p.filename}&conversation_id=${req.p.conversation_id} - -Please let us know if you have any questions about the data. - -Thanks for using Polis! -`; - - sendTextEmail(fromAddress, email, subject, body) - .then(function () { - res.status(200).json({}); - }) - .catch(function (err: any) { - failJson(res, 500, "polis_err_sending_export_link_to_email", err); - }); - } - function getLocationsForParticipants(zid: number) { return pg.queryP_readOnly( "select * from participant_locations where zid = ($1);", @@ -695,47 +568,6 @@ Thanks for using Polis! }); } - function handle_POST_contributors( - req: { - p: { - uid: null; - agreement_version: any; - name: any; - email: any; - github_id: any; - company_name: any; - }; - }, - res: { json: (arg0: {}) => void } - ) { - const uid = req.p.uid || null; - const agreement_version = req.p.agreement_version; - const name = req.p.name; - const email = req.p.email; - const github_id = req.p.github_id; - const company_name = req.p.company_name; - - pg.queryP( - "insert into contributor_agreement_signatures (uid, agreement_version, github_id, name, email, company_name) " + - "values ($1, $2, $3, $4, $5, $6);", - [uid, agreement_version, github_id, name, email, company_name] - ).then( - () => { - emailTeam( - "contributer agreement signed", - [uid, agreement_version, github_id, name, email, company_name].join( - "\n" - ) - ); - - res.json({}); - }, - (err: any) => { - failJson(res, 500, "polis_err_POST_contributors_misc", err); - } - ); - } - function handle_GET_testConnection( req: ExpressRequest, res: ExpressResponse @@ -828,10 +660,8 @@ Thanks for using Polis! handle_GET_verification, handle_GET_zinvites, handle_POST_contexts, - handle_POST_contributors, handle_POST_metrics, handle_POST_sendCreatedLinkToEmail, - handle_POST_sendEmailExportReady, handle_POST_tutorial, handle_POST_zinvites, }; diff --git a/server/src/utils/common.ts b/server/src/utils/common.ts index 73a192c271..d5b7b52d1a 100644 --- a/server/src/utils/common.ts +++ b/server/src/utils/common.ts @@ -36,27 +36,6 @@ function isPolisDev(uid?: any) { return polisDevs.indexOf(uid) >= 0; } -function strToHex(str: string) { - let hex, i; - // let str = "\u6f22\u5b57"; // "\u6f22\u5b57" === "漢字" - let result = ""; - for (i = 0; i < str.length; i++) { - hex = str.charCodeAt(i).toString(16); - result += ("000" + hex).slice(-4); - } - return result; -} - -function hexToStr(hexString: string) { - let j; - const hexes = hexString.match(/.{1,4}/g) || []; - let str = ""; - for (j = 0; j < hexes.length; j++) { - str += String.fromCharCode(parseInt(hexes[j], 16)); - } - return str; -} - const polisTypes: PolisTypes = { reactions: { push: 1, @@ -115,29 +94,6 @@ function isModerator(zid: number, uid?: number) { }); } -function doAddDataExportTask( - math_env: string | undefined, - email: string, - zid: number, - atDate: number, - format: string, - task_bucket: number -) { - return pg.queryP( - "insert into worker_tasks (math_env, task_data, task_type, task_bucket) values ($1, $2, 'generate_export_data', $3);", - [ - math_env, - { - email: email, - zid: zid, - "at-date": atDate, - format: format, - }, - task_bucket, // TODO hash the params to get a consistent number? - ] - ); -} - function isOwner(zid: number, uid: number) { return getConversationInfo(zid).then(function (info: any) { return info.owner === uid; @@ -188,9 +144,7 @@ function ifDefinedFirstElseSecond(first: any, second: boolean) { //todo: only one export export { - doAddDataExportTask, escapeLiteral, - hexToStr, ifDefinedFirstElseSecond, ifDefinedSet, isConversationOwner, @@ -200,18 +154,11 @@ export { isPolisDev, isUserAllowedToCreateConversations, polisTypes, - strToHex, }; export default { - doAddDataExportTask, escapeLiteral, - hexToStr, - isConversationOwner, - isDuplicateKey, isModerator, isOwner, - isPolisDev, polisTypes, - strToHex, }; From f8cc87ab476a3e8f22875c64215400f71d7dca0f Mon Sep 17 00:00:00 2001 From: Bennie Rosas Date: Sat, 25 Oct 2025 03:35:32 -0500 Subject: [PATCH 3/7] remove MATH_ENV from delphi; hard-code as prod --- delphi/example.env | 1 - delphi/polismath/__main__.py | 8 - delphi/polismath/components/config.py | 346 +- delphi/polismath/database/postgres.py | 7 +- delphi/schema.sql | 3512 +++++++++++++++++ .../801_narrative_report_batch.py | 80 +- 6 files changed, 3734 insertions(+), 220 deletions(-) create mode 100644 delphi/schema.sql diff --git a/delphi/example.env b/delphi/example.env index c896419f24..cf4eedd701 100644 --- a/delphi/example.env +++ b/delphi/example.env @@ -3,7 +3,6 @@ HOST=localhost PORT=8080 # Default WARN LOG_LEVEL=INFO -MATH_ENV=dev # LLM configuration # Default is ollama (alternative: anthropic) diff --git a/delphi/polismath/__main__.py b/delphi/polismath/__main__.py index 1fdcab346e..bf5b6c899c 100644 --- a/delphi/polismath/__main__.py +++ b/delphi/polismath/__main__.py @@ -57,11 +57,6 @@ def parse_args() -> argparse.Namespace: help='Directory for data files' ) - parser.add_argument( - '--math-env', - help='Math environment (dev, prod, preprod)' - ) - parser.add_argument( '--port', type=int, @@ -118,9 +113,6 @@ def main() -> None: if args.data_dir: overrides['data_dir'] = args.data_dir - if args.math_env: - overrides['math-env'] = args.math_env - if args.port: if 'server' not in overrides: overrides['server'] = {} diff --git a/delphi/polismath/components/config.py b/delphi/polismath/components/config.py index 27a5f92dd8..dcd0fc5e3d 100644 --- a/delphi/polismath/components/config.py +++ b/delphi/polismath/components/config.py @@ -21,16 +21,16 @@ def to_int(value: Any) -> Optional[int]: """ Convert a value to an integer. - + Args: value: Value to convert - + Returns: Integer value, or None if conversion failed """ if value is None: return None - + try: return int(value) except (ValueError, TypeError): @@ -40,16 +40,16 @@ def to_int(value: Any) -> Optional[int]: def to_float(value: Any) -> Optional[float]: """ Convert a value to a float. - + Args: value: Value to convert - + Returns: Float value, or None if conversion failed """ if value is None: return None - + try: return float(value) except (ValueError, TypeError): @@ -59,71 +59,71 @@ def to_float(value: Any) -> Optional[float]: def to_bool(value: Any) -> Optional[bool]: """ Convert a value to a boolean. - + Args: value: Value to convert - + Returns: Boolean value, or None if conversion failed """ if value is None: return None - + if isinstance(value, bool): return value - + if isinstance(value, (int, float)): return bool(value) - + if isinstance(value, str): value = value.lower().strip() - if value in ('true', 'yes', 'y', '1', 't'): + if value in ("true", "yes", "y", "1", "t"): return True - if value in ('false', 'no', 'n', '0', 'f'): + if value in ("false", "no", "n", "0", "f"): return False - + return None -def to_list(value: Any, separator: str = ',') -> Optional[List[str]]: +def to_list(value: Any, separator: str = ",") -> Optional[List[str]]: """ Convert a value to a list. - + Args: value: Value to convert separator: Separator for string values - + Returns: List value, or None if conversion failed """ if value is None: return None - + if isinstance(value, list): return value - + if isinstance(value, str): return [item.strip() for item in value.split(separator) if item.strip()] - + return None -def to_int_list(value: Any, separator: str = ',') -> Optional[List[int]]: +def to_int_list(value: Any, separator: str = ",") -> Optional[List[int]]: """ Convert a value to a list of integers. - + Args: value: Value to convert separator: Separator for string values - + Returns: List of integers, or None if conversion failed """ string_list = to_list(value, separator) - + if string_list is None: return None - + try: return [int(item) for item in string_list] except (ValueError, TypeError): @@ -133,11 +133,11 @@ def to_int_list(value: Any, separator: str = ',') -> Optional[List[int]]: def get_env_value(name: str, default: Any = None) -> Any: """ Get a value from environment variables. - + Args: name: Environment variable name default: Default value if not found - + Returns: Environment variable value, or default if not found """ @@ -148,151 +148,187 @@ class Config: """ Configuration manager for Pol.is math. """ - + def __init__(self, overrides: Optional[Dict[str, Any]] = None): """ Initialize configuration. - + Args: overrides: Optional configuration overrides """ self._lock = threading.RLock() self._config = {} self._initialized = False - + # Load configuration self.load_config(overrides) - + def load_config(self, overrides: Optional[Dict[str, Any]] = None) -> None: """ Load configuration from all sources. - + Args: overrides: Optional configuration overrides """ with self._lock: # Start with default configuration config = self._get_defaults() - + # Apply environment variables config = self._apply_env_vars(config) - + # Apply overrides if overrides: config = self._apply_overrides(config, overrides) - - # Apply inferred values - config = self._apply_inferred_values(config) - + # Store configuration self._config = config self._initialized = True - + logger.info("Configuration loaded") - + def _get_defaults(self) -> Dict[str, Any]: """ Get default configuration values. - + Returns: Default configuration """ return { - # Environment - 'math-env': 'dev', - # Server - 'server': { - 'port': 8080, - 'host': 'localhost' - }, - + "server": {"port": 8080, "host": "localhost"}, # Database - 'database': { - 'pool-size': 5, - 'max-overflow': 10 - }, - + "database": {"pool-size": 5, "max-overflow": 10}, # Polling - 'poller': { - 'vote-interval': 1.0, # seconds - 'mod-interval': 5.0, # seconds - 'task-interval': 10.0, # seconds - 'allowlist': [], # allowed conversation IDs - 'blocklist': [] # blocked conversation IDs + "poller": { + "vote-interval": 1.0, # seconds + "mod-interval": 5.0, # seconds + "task-interval": 10.0, # seconds + "allowlist": [], # allowed conversation IDs + "blocklist": [], # blocked conversation IDs }, - # Conversation - 'conversation': { - 'max-ptpts': 5000, # maximum participants - 'max-cmts': 400, # maximum comments - 'group-k-min': 2, # minimum number of groups - 'group-k-max': 5 # maximum number of groups + "conversation": { + "max-ptpts": 5000, # maximum participants + "max-cmts": 400, # maximum comments + "group-k-min": 2, # minimum number of groups + "group-k-max": 5, # maximum number of groups }, - # Logging - 'logging': { - 'level': 'warn' - } + "logging": {"level": "warn"}, } - + def _apply_env_vars(self, config: Dict[str, Any]) -> Dict[str, Any]: """ Apply environment variables to configuration. - + Args: config: Current configuration - + Returns: Updated configuration """ # Make a copy config = deepcopy(config) - - # Environment - if 'MATH_ENV' in os.environ: - config['math-env'] = os.environ['MATH_ENV'] - + # Server - config['server']['port'] = to_int(os.environ.get('PORT', config['server']['port'])) - config['server']['host'] = os.environ.get('HOST', config['server']['host']) - + config["server"]["port"] = to_int( + os.environ.get("PORT", config["server"]["port"]) + ) + config["server"]["host"] = os.environ.get("HOST", config["server"]["host"]) + # Database - config['database']['pool-size'] = to_int(os.environ.get('DATABASE_POOL_SIZE', config['database']['pool-size'])) - config['database']['max-overflow'] = to_int(os.environ.get('DATABASE_MAX_OVERFLOW', config['database']['max-overflow'])) - + config["database"]["pool-size"] = to_int( + os.environ.get("DATABASE_POOL_SIZE", config["database"]["pool-size"]) + ) + config["database"]["max-overflow"] = to_int( + os.environ.get("DATABASE_MAX_OVERFLOW", config["database"]["max-overflow"]) + ) + # Polling - config['poller']['vote-interval'] = to_float(os.environ.get('POLL_VOTE_INTERVAL_MS', to_float(os.environ.get('POLL_INTERVAL_MS', config['poller']['vote-interval'] * 1000)))) / 1000.0 - config['poller']['mod-interval'] = to_float(os.environ.get('POLL_MOD_INTERVAL_MS', to_float(os.environ.get('POLL_INTERVAL_MS', config['poller']['mod-interval'] * 1000)))) / 1000.0 - config['poller']['task-interval'] = to_float(os.environ.get('POLL_TASK_INTERVAL_MS', to_float(os.environ.get('POLL_INTERVAL_MS', config['poller']['task-interval'] * 1000)))) / 1000.0 - config['poller']['allowlist'] = to_int_list(os.environ.get('POLL_ALLOWLIST', [])) - config['poller']['blocklist'] = to_int_list(os.environ.get('POLL_BLOCKLIST', [])) - + config["poller"]["vote-interval"] = ( + to_float( + os.environ.get( + "POLL_VOTE_INTERVAL_MS", + to_float( + os.environ.get( + "POLL_INTERVAL_MS", config["poller"]["vote-interval"] * 1000 + ) + ), + ) + ) + / 1000.0 + ) + config["poller"]["mod-interval"] = ( + to_float( + os.environ.get( + "POLL_MOD_INTERVAL_MS", + to_float( + os.environ.get( + "POLL_INTERVAL_MS", config["poller"]["mod-interval"] * 1000 + ) + ), + ) + ) + / 1000.0 + ) + config["poller"]["task-interval"] = ( + to_float( + os.environ.get( + "POLL_TASK_INTERVAL_MS", + to_float( + os.environ.get( + "POLL_INTERVAL_MS", config["poller"]["task-interval"] * 1000 + ) + ), + ) + ) + / 1000.0 + ) + config["poller"]["allowlist"] = to_int_list( + os.environ.get("POLL_ALLOWLIST", []) + ) + config["poller"]["blocklist"] = to_int_list( + os.environ.get("POLL_BLOCKLIST", []) + ) + # Conversation - config['conversation']['max-ptpts'] = to_int(os.environ.get('CONV_MAX_PTPTS', config['conversation']['max-ptpts'])) - config['conversation']['max-cmts'] = to_int(os.environ.get('CONV_MAX_CMTS', config['conversation']['max-cmts'])) - config['conversation']['group-k-min'] = to_int(os.environ.get('CONV_GROUP_K_MIN', config['conversation']['group-k-min'])) - config['conversation']['group-k-max'] = to_int(os.environ.get('CONV_GROUP_K_MAX', config['conversation']['group-k-max'])) - + config["conversation"]["max-ptpts"] = to_int( + os.environ.get("CONV_MAX_PTPTS", config["conversation"]["max-ptpts"]) + ) + config["conversation"]["max-cmts"] = to_int( + os.environ.get("CONV_MAX_CMTS", config["conversation"]["max-cmts"]) + ) + config["conversation"]["group-k-min"] = to_int( + os.environ.get("CONV_GROUP_K_MIN", config["conversation"]["group-k-min"]) + ) + config["conversation"]["group-k-max"] = to_int( + os.environ.get("CONV_GROUP_K_MAX", config["conversation"]["group-k-max"]) + ) + # Logging - config['logging']['level'] = os.environ.get('LOG_LEVEL', config['logging']['level']).lower() - + config["logging"]["level"] = os.environ.get( + "LOG_LEVEL", config["logging"]["level"] + ).lower() + return config - - def _apply_overrides(self, config: Dict[str, Any], overrides: Dict[str, Any]) -> Dict[str, Any]: + + def _apply_overrides( + self, config: Dict[str, Any], overrides: Dict[str, Any] + ) -> Dict[str, Any]: """ Apply configuration overrides. - + Args: config: Current configuration overrides: Configuration overrides - + Returns: Updated configuration """ # Make a copy config = deepcopy(config) - + # Helper function for deep update def deep_update(d, u): for k, v in u.items(): @@ -301,69 +337,43 @@ def deep_update(d, u): else: d[k] = v return d - + # Apply overrides return deep_update(config, overrides) - - def _apply_inferred_values(self, config: Dict[str, Any]) -> Dict[str, Any]: - """ - Apply inferred configuration values. - - Args: - config: Current configuration - - Returns: - Updated configuration - """ - # Make a copy - config = deepcopy(config) - - # Set math-env-string - config['math-env-string'] = str(config['math-env']) - - # Set webserver-url based on environment - if config['math-env'] == 'prod': - config['webserver-url'] = "https://pol.is" - elif config['math-env'] == 'preprod': - config['webserver-url'] = "https://preprod.pol.is" - else: - config['webserver-url'] = f"http://{config['server']['host']}:{config['server']['port']}" - - return config - + def get(self, path: str, default: Any = None) -> Any: """ Get a configuration value. - + Args: path: Configuration path (dot-separated) default: Default value if not found - + Returns: Configuration value, or default if not found """ if not self._initialized: self.load_config() - + # Split path into components - components = path.split('.') - + components = path.split(".") + # Start with full configuration value = self._config - + # Traverse path for component in components: if isinstance(value, dict) and component in value: value = value[component] else: return default - + return value - + def set(self, path: str, value: Any) -> None: """ Set a configuration value. - + Args: path: Configuration path (dot-separated) value: Configuration value @@ -371,72 +381,72 @@ def set(self, path: str, value: Any) -> None: with self._lock: if not self._initialized: self.load_config() - + # Split path into components - components = path.split('.') - + components = path.split(".") + # Start with full configuration config = self._config - + # Traverse path for i, component in enumerate(components[:-1]): if component not in config: config[component] = {} - + config = config[component] - + # Set value config[components[-1]] = value - + def to_dict(self) -> Dict[str, Any]: """ Convert configuration to a dictionary. - + Returns: Configuration dictionary """ if not self._initialized: self.load_config() - + return deepcopy(self._config) - + def save_to_file(self, filepath: str) -> None: """ Save configuration to a file. - + Args: filepath: Path to save configuration """ if not self._initialized: self.load_config() - + # Determine file format from extension - if filepath.endswith('.json'): - with open(filepath, 'w') as f: + if filepath.endswith(".json"): + with open(filepath, "w") as f: json.dump(self._config, f, indent=2) - elif filepath.endswith('.yaml') or filepath.endswith('.yml'): - with open(filepath, 'w') as f: + elif filepath.endswith(".yaml") or filepath.endswith(".yml"): + with open(filepath, "w") as f: yaml.dump(self._config, f, default_flow_style=False) else: raise ValueError(f"Unsupported file format: {filepath}") - + def load_from_file(self, filepath: str) -> None: """ Load configuration from a file. - + Args: filepath: Path to load configuration from """ # Determine file format from extension - if filepath.endswith('.json'): - with open(filepath, 'r') as f: + if filepath.endswith(".json"): + with open(filepath, "r") as f: overrides = json.load(f) - elif filepath.endswith('.yaml') or filepath.endswith('.yml'): - with open(filepath, 'r') as f: + elif filepath.endswith(".yaml") or filepath.endswith(".yml"): + with open(filepath, "r") as f: overrides = yaml.safe_load(f) else: raise ValueError(f"Unsupported file format: {filepath}") - + # Apply overrides self.load_config(overrides) @@ -445,18 +455,18 @@ class ConfigManager: """ Singleton manager for configuration. """ - + _instance = None _lock = threading.RLock() - + @classmethod def get_config(cls, overrides: Optional[Dict[str, Any]] = None) -> Config: """ Get the configuration instance. - + Args: overrides: Optional configuration overrides - + Returns: Config instance """ @@ -465,5 +475,5 @@ def get_config(cls, overrides: Optional[Dict[str, Any]] = None) -> Config: cls._instance = Config(overrides) elif overrides: cls._instance.load_config(overrides) - - return cls._instance \ No newline at end of file + + return cls._instance diff --git a/delphi/polismath/database/postgres.py b/delphi/polismath/database/postgres.py index cc17161bc5..4fb71dd395 100644 --- a/delphi/polismath/database/postgres.py +++ b/delphi/polismath/database/postgres.py @@ -50,7 +50,6 @@ def __init__( pool_size: Optional[int] = None, max_overflow: Optional[int] = None, ssl_mode: Optional[str] = None, - math_env: Optional[str] = None, ): """ Initialize PostgreSQL configuration. @@ -65,7 +64,6 @@ def __init__( pool_size: Connection pool size max_overflow: Maximum overflow connections ssl_mode: SSL mode (disable, allow, prefer, require, verify-ca, verify-full) - math_env: Math environment identifier """ # Parse URL if provided if url: @@ -89,8 +87,8 @@ def __init__( # Set SSL mode self.ssl_mode = ssl_mode or os.environ.get("DATABASE_SSL_MODE", "require") - # Set math environment - self.math_env = math_env or os.environ.get("MATH_ENV", "dev") + # Hardcode math_env to "prod" + self.math_env = "prod" def _parse_url(self, url: str) -> None: """ @@ -159,7 +157,6 @@ def from_env(cls) -> "PostgresConfig": database=os.environ.get("DATABASE_NAME"), user=os.environ.get("DATABASE_USER"), password=os.environ.get("DATABASE_PASSWORD"), - math_env=os.environ.get("MATH_ENV"), ) diff --git a/delphi/schema.sql b/delphi/schema.sql new file mode 100644 index 0000000000..0d1d2d018e --- /dev/null +++ b/delphi/schema.sql @@ -0,0 +1,3512 @@ +-- +-- PostgreSQL database dump +-- + +\restrict yi3q8cVeyPYXP7xpzDDwpdv2ts6MbtdVFjxHM5kY0sd3iTqn6dEoZ2y0ASHXmgp + +-- Dumped from database version 17.6 +-- Dumped by pg_dump version 18.0 + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET transaction_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: pg_stat_statements; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pg_stat_statements WITH SCHEMA public; + + +-- +-- Name: EXTENSION pg_stat_statements; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pg_stat_statements IS 'track execution statistics of all SQL statements executed'; + + +-- +-- Name: tablefunc; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS tablefunc WITH SCHEMA public; + + +-- +-- Name: EXTENSION tablefunc; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION tablefunc IS 'functions that manipulate whole tables, including crosstab'; + + +-- +-- Name: animal_grp; Type: TYPE; Schema: public; Owner: - +-- + +CREATE TYPE public.animal_grp AS ENUM ( + 'fish', + 'mammal', + 'bird' +); + + +-- +-- Name: animals_id_auto(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.animals_id_auto() RETURNS trigger + LANGUAGE plpgsql STRICT + AS $$ +DECLARE + _rel_id constant int := 'animals'::regclass::int; + _grp_id int; +BEGIN + _grp_id = array_length(enum_range(NULL, NEW.grp), 1); + + -- Obtain an advisory lock on this table/group. + PERFORM pg_advisory_lock(_rel_id, _grp_id); + + SELECT COALESCE(MAX(id) + 1, 1) + INTO NEW.id + FROM animals + WHERE grp = NEW.grp; + + RETURN NEW; +END; +$$; + + +-- +-- Name: get_times_for_most_recent_visible_comments(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.get_times_for_most_recent_visible_comments() RETURNS TABLE(zid integer, modified bigint) + LANGUAGE sql + AS $$ + select zid, max(modified) from (select comments.*, conversations.strict_moderation from comments left join conversations on comments.zid = conversations.zid) as c where c.mod >= (CASE WHEN c.strict_moderation=TRUE then 1 else 0 END) group by c.zid order by c.zid; +$$; + + +-- +-- Name: get_visible_comments(integer); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.get_visible_comments(the_zid integer) RETURNS TABLE(tid integer, mod integer, strict_moderation boolean) + LANGUAGE sql + AS $$ + select comments.tid, comments.mod, conversations.strict_moderation from comments left join conversations on comments.zid = conversations.zid where active = true and mod >= (CASE WHEN strict_moderation=TRUE then 1 else 0 END) and comments.zid = the_zid; +$$; + + +-- +-- Name: now_as_millis(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.now_as_millis() RETURNS bigint + LANGUAGE plpgsql + AS $$ + DECLARE + temp TIMESTAMP := now(); + BEGIN + -- NOTE: milliseconds includes the seconds, so subtracting seconds from milliseconds + -- SEE: http://www.postgresql.org/docs/8.4/static/functions-datetime.html + RETURN 1000*FLOOR(EXTRACT(EPOCH FROM temp)) + FLOOR(EXTRACT(MILLISECONDS FROM temp)) - 1000*FLOOR(EXTRACT(SECOND FROM temp)); + END; +$$; + + +-- +-- Name: oid_auto(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.oid_auto() RETURNS trigger + LANGUAGE plpgsql STRICT + AS $$ +DECLARE + _magic_id constant int := 873791984; -- This is a magic key used for locking conversation row-sets within the opinions table. TODO keep track of these + _opinion_id int; +BEGIN + _opinion_id = NEW.oid; + + -- Obtain an advisory lock on the opinions table, limited to this conversation + PERFORM pg_advisory_lock(_magic_id, _opinion_id); + + SELECT COALESCE(MAX(oid) + 1, 1) + INTO NEW.oid + FROM opinions + WHERE oid = NEW.oid; + + RETURN NEW; +END; +$$; + + +-- +-- Name: oid_auto_unlock(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.oid_auto_unlock() RETURNS trigger + LANGUAGE plpgsql STRICT + AS $$ +DECLARE + _magic_id constant int := 873791984; + _opinion_id int; +BEGIN + _opinion_id = NEW.oid; + + -- Release the lock. + PERFORM pg_advisory_unlock(_magic_id, _opinion_id); + + RETURN NEW; +END; +$$; + + +-- +-- Name: pid_auto(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.pid_auto() RETURNS trigger + LANGUAGE plpgsql STRICT + AS $$ +DECLARE + _magic_id constant int := 873791983; -- This is a magic key used for locking conversation row-sets within the participants table. TODO keep track of these + _conversation_id int; +BEGIN + _conversation_id = NEW.zid; + + -- Obtain an advisory lock on the participants table, limited to this conversation + PERFORM pg_advisory_lock(_magic_id, _conversation_id); + + SELECT COALESCE(MAX(pid) + 1, 0) -- Start with comment id of 0 + INTO NEW.pid + FROM participants + WHERE zid = NEW.zid; + + -- Duplicate participant_count to the conversations table to speed up conversationsView queries. + UPDATE conversations + SET participant_count = NEW.pid + 1 + WHERE zid = NEW.zid; + + RETURN NEW; +END; +$$; + + +-- +-- Name: pid_auto_unlock(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.pid_auto_unlock() RETURNS trigger + LANGUAGE plpgsql STRICT + AS $$ +DECLARE + _magic_id constant int := 873791983; + _conversation_id int; +BEGIN + _conversation_id = NEW.zid; + + -- Release the lock. + PERFORM pg_advisory_unlock(_magic_id, _conversation_id); + + RETURN NEW; +END; +$$; + + +-- +-- Name: ptpt_id_auto(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.ptpt_id_auto() RETURNS trigger + LANGUAGE plpgsql STRICT + AS $$ +DECLARE + _rel_id constant int := 873791983; -- This is a magic key used for locking conversation row-sets within the participants table. TODO keep track of these + _grp_id int; +BEGIN + _grp_id = NEW.conv_id; + + -- Obtain an advisory lock on the participants table, limited to this conversation + PERFORM pg_advisory_lock(_rel_id, _grp_id); + + SELECT COALESCE(MAX(ptpt_id) + 1, 1) + INTO NEW.ptpt_id + FROM participants + WHERE conv_id = NEW.conv_id; + + RETURN NEW; +END; +$$; + + +-- +-- Name: ptpt_id_auto_unlock(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.ptpt_id_auto_unlock() RETURNS trigger + LANGUAGE plpgsql STRICT + AS $$ +DECLARE + _rel_id constant int := 873791983; + _grp_id int; +BEGIN + _grp_id = NEW.conv_id; + + -- Release the lock. + PERFORM pg_advisory_unlock(_rel_id, _grp_id); + + RETURN NEW; +END; +$$; + + +-- +-- Name: random_polis_site_id(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.random_polis_site_id() RETURNS text + LANGUAGE sql + AS $$ +SELECT 'polis_site_id_' || random_string(18); +$$; + + +-- +-- Name: random_polis_site_id(integer); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.random_polis_site_id(integer) RETURNS text + LANGUAGE sql + AS $$ +SELECT 'polis_site_id_' || random_string(18); +$$; + + +-- +-- Name: random_string(integer); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.random_string(integer) RETURNS text + LANGUAGE sql + AS $_$ +SELECT array_to_string( + ARRAY ( + SELECT substring( + '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' + FROM (ceil(random()*62))::int FOR 1 + ) + FROM generate_series(1, $1) + ), + '' +) +$_$; + + +-- +-- Name: tid_auto(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.tid_auto() RETURNS trigger + LANGUAGE plpgsql STRICT + AS $$ +DECLARE + _magic_id constant int := 873791984; -- This is a magic key used for locking conversation row-sets within the comments table. TODO keep track of these + _conversation_id int; +BEGIN + _conversation_id = NEW.zid; + + -- Obtain an advisory lock on the comments table, limited to this conversation + PERFORM pg_advisory_lock(_magic_id, _conversation_id); + + SELECT COALESCE(MAX(tid) + 1, 0) -- Start with comment id of 0 + INTO NEW.tid + FROM comments + WHERE zid = NEW.zid; + + RETURN NEW; +END; +$$; + + +-- +-- Name: tid_auto_unlock(); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.tid_auto_unlock() RETURNS trigger + LANGUAGE plpgsql STRICT + AS $$ +DECLARE + _magic_id constant int := 873791984; + _conversation_id int; +BEGIN + _conversation_id = NEW.zid; + + -- Release the lock. + PERFORM pg_advisory_unlock(_magic_id, _conversation_id); + + RETURN NEW; +END; +$$; + + +-- +-- Name: to_millis(timestamp with time zone); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.to_millis(t timestamp with time zone) RETURNS bigint + LANGUAGE plpgsql + AS $$ + BEGIN + RETURN 1000*FLOOR(EXTRACT(EPOCH FROM t)) + FLOOR(EXTRACT(MILLISECONDS FROM t)) - 1000*FLOOR(EXTRACT(SECOND FROM t)); + END; +$$; + + +-- +-- Name: to_zid(text); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.to_zid(associated_zinvite text) RETURNS integer + LANGUAGE plpgsql + AS $$ + BEGIN + RETURN (select zid from zinvites where zinvite = associated_zinvite); + END; +$$; + + +-- +-- Name: to_zinvite(integer); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.to_zinvite(associated_zid integer) RETURNS text + LANGUAGE plpgsql + AS $$ + BEGIN + RETURN (select zinvite from zinvites where zid = associated_zid); + END; +$$; + + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: votes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.votes ( + zid integer NOT NULL, + pid integer NOT NULL, + tid integer NOT NULL, + vote smallint, + created bigint DEFAULT public.now_as_millis(), + weight_x_32767 smallint DEFAULT 0, + high_priority boolean DEFAULT false NOT NULL +); + + +-- +-- Name: votes_foo(integer); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.votes_foo(integer) RETURNS SETOF public.votes + LANGUAGE sql + AS $_$ + select * from votes where zid = $1; +$_$; + + +-- +-- Name: votes_lastest_unique(integer); Type: FUNCTION; Schema: public; Owner: - +-- + +CREATE FUNCTION public.votes_lastest_unique(integer) RETURNS SETOF public.votes + LANGUAGE sql + AS $_$ + WITH m AS (SELECT zid, pid, tid, MAX(created) AS created FROM votes WHERE zid = $1 GROUP BY zid, pid, tid) SELECT v.* FROM m LEFT JOIN votes v ON m.zid = v.zid AND m.pid = v.pid AND m.tid = v.tid WHERE m.created = v.created; +$_$; + + +-- +-- Name: apikeysndvweifu; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.apikeysndvweifu ( + uid integer NOT NULL, + apikey character varying(32) NOT NULL, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: auth_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.auth_tokens ( + token character varying(32), + uid integer, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: beta; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.beta ( + name character varying(999), + email character varying(200), + organization character varying(200), + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: comment_translations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.comment_translations ( + zid integer NOT NULL, + tid integer NOT NULL, + src integer NOT NULL, + txt character varying(9999) NOT NULL, + lang character varying(10) NOT NULL, + created bigint DEFAULT public.now_as_millis(), + modified bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: comments; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.comments ( + tid integer NOT NULL, + zid integer NOT NULL, + pid integer NOT NULL, + txt character varying(1000) NOT NULL, + created bigint DEFAULT public.now_as_millis(), + velocity real DEFAULT 1, + mod integer DEFAULT 0 NOT NULL, + active boolean DEFAULT true NOT NULL, + modified bigint DEFAULT public.now_as_millis(), + uid integer DEFAULT 0 NOT NULL, + tweet_id bigint, + quote_src_url character varying(1000), + anon boolean DEFAULT false NOT NULL, + is_seed boolean DEFAULT false NOT NULL, + curation smallint DEFAULT 0 NOT NULL, + is_meta boolean DEFAULT false NOT NULL, + lang_confidence real, + lang character varying(10) +); + + +-- +-- Name: contexts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.contexts ( + context_id integer NOT NULL, + name character varying(300), + creator integer, + is_public boolean DEFAULT false, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: contexts_context_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.contexts_context_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: contexts_context_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.contexts_context_id_seq OWNED BY public.contexts.context_id; + + +-- +-- Name: contributor_agreement_signatures; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.contributor_agreement_signatures ( + uid integer, + name character varying(746) NOT NULL, + github_id character varying(256), + email character varying(256) NOT NULL, + agreement_version integer NOT NULL, + created bigint DEFAULT public.now_as_millis(), + company_name character varying(746) +); + + +-- +-- Name: conversation_invite_codes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.conversation_invite_codes ( + zid integer NOT NULL, + code character varying(300) NOT NULL, + created timestamp with time zone DEFAULT now() +); + + +-- +-- Name: conversation_subscriptions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.conversation_subscriptions ( + zid integer NOT NULL, + uid integer NOT NULL, + time_last_notified bigint DEFAULT 0, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: conversation_translations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.conversation_translations ( + zid integer NOT NULL, + src integer NOT NULL, + topic character varying(9999) NOT NULL, + description character varying(9999) NOT NULL, + lang character varying(10) NOT NULL, + created bigint DEFAULT public.now_as_millis(), + modified bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: conversations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.conversations ( + zid integer NOT NULL, + topic character varying(1000), + description character varying(50000), + is_anon boolean DEFAULT true, + is_active boolean DEFAULT false, + is_draft boolean DEFAULT false, + is_public boolean DEFAULT true, + email_domain character varying(200), + owner integer, + participant_count integer DEFAULT 0, + created bigint DEFAULT public.now_as_millis(), + strict_moderation boolean DEFAULT false, + profanity_filter boolean DEFAULT true, + spam_filter boolean DEFAULT true, + context character varying(1000), + modified bigint DEFAULT public.now_as_millis(), + owner_sees_participation_stats boolean DEFAULT false, + course_id integer, + link_url character varying(9999), + upvotes integer DEFAULT 1 NOT NULL, + parent_url character varying(9999), + vis_type integer DEFAULT 0 NOT NULL, + write_type integer DEFAULT 1 NOT NULL, + bgcolor character varying(20), + help_type integer DEFAULT 1 NOT NULL, + socialbtn_type integer DEFAULT 0 NOT NULL, + style_btn character varying(500), + auth_needed_to_vote boolean DEFAULT false, + auth_needed_to_write boolean DEFAULT true, + auth_opt_fb boolean DEFAULT true, + auth_opt_tw boolean DEFAULT true, + auth_opt_allow_3rdparty boolean DEFAULT true, + help_bgcolor character varying(20), + help_color character varying(20), + is_data_open boolean DEFAULT false, + is_curated boolean DEFAULT false, + dataset_explanation character varying(50000), + write_hint_type integer DEFAULT 1 NOT NULL, + subscribe_type integer DEFAULT 1 NOT NULL, + org_id integer, + need_suzinvite boolean DEFAULT false, + use_xid_whitelist boolean DEFAULT false, + prioritize_seed boolean DEFAULT false, + importance_enabled boolean DEFAULT false NOT NULL, + treevite_enabled boolean DEFAULT false +); + + +-- +-- Name: COLUMN conversations.treevite_enabled; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.conversations.treevite_enabled IS 'Enable wave-based invite (Treevite) for this conversation'; + + +-- +-- Name: conversations_zid_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.conversations_zid_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: conversations_zid_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.conversations_zid_seq OWNED BY public.conversations.zid; + + +-- +-- Name: courses; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.courses ( + course_id integer NOT NULL, + topic character varying(1000), + description character varying(1000), + owner integer, + created bigint DEFAULT public.now_as_millis(), + course_invite character varying(32) +); + + +-- +-- Name: courses_course_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.courses_course_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: courses_course_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.courses_course_id_seq OWNED BY public.courses.course_id; + + +-- +-- Name: crowd_mod; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.crowd_mod ( + zid integer NOT NULL, + pid integer NOT NULL, + tid integer NOT NULL, + created bigint DEFAULT public.now_as_millis(), + as_important boolean, + as_factual boolean, + as_feeling boolean, + as_notmyfeeling boolean, + as_notgoodidea boolean, + as_notfact boolean, + as_unsure boolean, + as_spam boolean, + as_abusive boolean, + as_offtopic boolean +); + + +-- +-- Name: demographic_data; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.demographic_data ( + uid integer, + fb_gender integer, + ms_birth_year_estimate_fb integer, + ms_gender_estimate_fb integer, + fb_timestamp bigint DEFAULT public.now_as_millis(), + ms_fb_timestamp bigint DEFAULT public.now_as_millis(), + ms_response character varying(9999), + gender_guess integer, + birth_year_guess integer +); + + +-- +-- Name: einvites; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.einvites ( + einvite character varying(100) NOT NULL, + email character varying(999), + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: email_validations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.email_validations ( + email character varying(999), + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: error_reports; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.error_reports ( + uid integer NOT NULL, + zid integer NOT NULL, + error_code character varying(99), + data character varying(9999), + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: event_ptpt_no_more_comments; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.event_ptpt_no_more_comments ( + zid integer NOT NULL, + pid integer NOT NULL, + votes_placed smallint NOT NULL, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: facebook_friends; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.facebook_friends ( + uid integer NOT NULL, + friend integer NOT NULL +); + + +-- +-- Name: facebook_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.facebook_users ( + uid integer NOT NULL, + fb_user_id text, + fb_public_profile text, + fb_login_status text, + fb_auth_response text, + fb_access_token text, + fb_granted_scopes text, + response text, + fb_friends_response text, + created bigint DEFAULT public.now_as_millis(), + location character varying(9999), + fb_location_id character varying(100), + fb_name character varying(9999), + fb_link character varying(9999), + modified bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: foobar; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.foobar + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: inviters; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.inviters ( + inviter_uid integer, + invited_email character varying(999), + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: jianiuevyew; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.jianiuevyew ( + uid integer NOT NULL, + pwhash character varying(128) NOT NULL +); + + +-- +-- Name: math_bidtopid; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.math_bidtopid ( + zid integer NOT NULL, + math_env character varying(999) NOT NULL, + data json NOT NULL, + modified bigint DEFAULT public.now_as_millis(), + math_tick bigint DEFAULT '-1'::integer NOT NULL +); + + +-- +-- Name: math_cache; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.math_cache ( + zid integer NOT NULL, + math_env character varying(999) NOT NULL, + data json NOT NULL, + modified bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: math_exportstatus; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.math_exportstatus ( + zid integer NOT NULL, + math_env character varying(999) NOT NULL, + filename character varying(9999) NOT NULL, + data json NOT NULL, + modified bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: math_main; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.math_main ( + zid integer NOT NULL, + math_env character varying(999) NOT NULL, + data json NOT NULL, + last_vote_timestamp bigint NOT NULL, + modified bigint DEFAULT public.now_as_millis(), + math_tick bigint DEFAULT '-1'::integer NOT NULL, + caching_tick bigint DEFAULT 0 NOT NULL +); + + +-- +-- Name: math_profile; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.math_profile ( + zid integer NOT NULL, + math_env character varying(999) NOT NULL, + data json NOT NULL, + modified bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: math_ptptstats; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.math_ptptstats ( + zid integer NOT NULL, + math_env character varying(999) NOT NULL, + data json NOT NULL, + modified bigint DEFAULT public.now_as_millis(), + math_tick bigint DEFAULT '-1'::integer NOT NULL +); + + +-- +-- Name: math_report_correlationmatrix; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.math_report_correlationmatrix ( + rid bigint NOT NULL, + math_env character varying(999) NOT NULL, + data jsonb, + modified bigint DEFAULT public.now_as_millis(), + math_tick bigint DEFAULT '-1'::integer NOT NULL +); + + +-- +-- Name: math_results_dev01; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.math_results_dev01 ( + zid integer NOT NULL, + last_timestamp bigint NOT NULL, + data text +); + + +-- +-- Name: math_ticks; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.math_ticks ( + zid integer, + math_tick bigint DEFAULT 0 NOT NULL, + math_env character varying(999) NOT NULL, + modified bigint DEFAULT public.now_as_millis() NOT NULL +); + + +-- +-- Name: metrics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.metrics ( + uid integer, + type integer NOT NULL, + dur integer, + created bigint DEFAULT public.now_as_millis(), + hashedpc integer +); + + +-- +-- Name: minvites; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.minvites ( + zid integer NOT NULL, + minvite character varying(300) NOT NULL, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: moderators; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.moderators ( + zid integer NOT NULL, + uid integer, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: notification_tasks; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.notification_tasks ( + zid integer NOT NULL, + modified bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: nyt_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.nyt_users ( + uid integer NOT NULL, + nyt_user_id bigint NOT NULL, + nyt_name character varying(9999), + nyt_img character varying(9999), + modified bigint DEFAULT public.now_as_millis() NOT NULL, + created bigint DEFAULT public.now_as_millis() NOT NULL +); + + +-- +-- Name: oidc_user_mappings; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.oidc_user_mappings ( + oidc_sub character varying(255) NOT NULL, + uid integer NOT NULL, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: COLUMN oidc_user_mappings.oidc_sub; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.oidc_user_mappings.oidc_sub IS 'OIDC subject (sub) claim from JWT'; + + +-- +-- Name: COLUMN oidc_user_mappings.uid; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.oidc_user_mappings.uid IS 'Local Polis user ID'; + + +-- +-- Name: COLUMN oidc_user_mappings.created; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.oidc_user_mappings.created IS 'Timestamp when mapping was created'; + + +-- +-- Name: oinvites; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.oinvites ( + oinvite character varying(300) NOT NULL, + note character varying(999), + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: page_ids; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.page_ids ( + site_id character varying(100) NOT NULL, + page_id character varying(100) NOT NULL, + zid integer NOT NULL +); + + +-- +-- Name: participant_locations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.participant_locations ( + zid integer NOT NULL, + uid integer NOT NULL, + pid integer NOT NULL, + lat double precision NOT NULL, + lng double precision NOT NULL, + created bigint DEFAULT public.now_as_millis(), + source integer NOT NULL +); + + +-- +-- Name: participant_metadata_answers; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.participant_metadata_answers ( + pmaid integer NOT NULL, + pmqid integer, + zid integer, + value character varying(999), + alive boolean DEFAULT true, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: participant_metadata_answers_pmaid_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.participant_metadata_answers_pmaid_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: participant_metadata_answers_pmaid_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.participant_metadata_answers_pmaid_seq OWNED BY public.participant_metadata_answers.pmaid; + + +-- +-- Name: participant_metadata_choices; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.participant_metadata_choices ( + zid integer, + pid integer, + pmqid integer, + pmaid integer, + alive boolean DEFAULT true, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: participant_metadata_questions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.participant_metadata_questions ( + pmqid integer NOT NULL, + zid integer, + key character varying(999), + alive boolean DEFAULT true, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: participant_metadata_questions_pmqid_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.participant_metadata_questions_pmqid_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: participant_metadata_questions_pmqid_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.participant_metadata_questions_pmqid_seq OWNED BY public.participant_metadata_questions.pmqid; + + +-- +-- Name: participants; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.participants ( + pid integer NOT NULL, + uid integer NOT NULL, + zid integer NOT NULL, + created bigint DEFAULT public.now_as_millis(), + vote_count integer DEFAULT 0 NOT NULL, + last_interaction bigint DEFAULT 0 NOT NULL, + subscribed integer DEFAULT 0 NOT NULL, + last_notified bigint DEFAULT 0, + mod integer DEFAULT 0 NOT NULL, + nsli smallint DEFAULT 0 NOT NULL +); + + +-- +-- Name: participants_extended; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.participants_extended ( + uid integer NOT NULL, + zid integer NOT NULL, + referrer character varying(9999), + parent_url character varying(9999), + created bigint DEFAULT public.now_as_millis(), + permanent_cookie character varying(32), + origin character varying(9999), + encrypted_ip_address character varying(9999), + encrypted_x_forwarded_for character varying(9999), + country_iso_code character varying(10), + modified bigint DEFAULT public.now_as_millis() NOT NULL, + show_translation_activated boolean, + subscribe_email character varying(256) +); + + +-- +-- Name: permanentcookiezidjoins; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.permanentcookiezidjoins ( + zid integer NOT NULL, + cookie character varying(32), + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: polismath_mod_claims; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.polismath_mod_claims ( + n integer NOT NULL, + created bigint DEFAULT public.now_as_millis() NOT NULL +); + + +-- +-- Name: pwreset_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.pwreset_tokens ( + token character varying(100), + uid integer, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: queue; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.queue ( + itemid integer NOT NULL, + is_done boolean DEFAULT false NOT NULL +); + + +-- +-- Name: report_comment_selections; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.report_comment_selections ( + rid bigint NOT NULL, + tid integer NOT NULL, + selection smallint NOT NULL, + modified bigint DEFAULT public.now_as_millis(), + zid integer NOT NULL +); + + +-- +-- Name: reports; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.reports ( + rid bigint NOT NULL, + report_id character varying(300) NOT NULL, + zid integer NOT NULL, + created bigint DEFAULT public.now_as_millis(), + modified bigint DEFAULT public.now_as_millis(), + label_x_neg character varying(999), + label_y_neg character varying(999), + label_y_pos character varying(999), + label_x_pos character varying(999), + label_group_0 character varying(999), + label_group_1 character varying(999), + label_group_2 character varying(999), + label_group_3 character varying(999), + label_group_4 character varying(999), + label_group_5 character varying(999), + label_group_6 character varying(999), + label_group_7 character varying(999), + label_group_8 character varying(999), + label_group_9 character varying(999), + report_name character varying(999), + mod_level smallint DEFAULT '-2'::integer NOT NULL +); + + +-- +-- Name: reports_rid_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.reports_rid_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: reports_rid_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.reports_rid_seq OWNED BY public.reports.rid; + + +-- +-- Name: site_domain_whitelist; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.site_domain_whitelist ( + site_id character varying(256) NOT NULL, + domain_whitelist character varying(999), + modified bigint DEFAULT public.now_as_millis() NOT NULL, + created bigint DEFAULT public.now_as_millis() NOT NULL, + domain_whitelist_override_key character varying(999) +); + + +-- +-- Name: slack_participants_waiting_for_comments; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.slack_participants_waiting_for_comments ( + zid integer NOT NULL, + uid integer NOT NULL, + slack_team character varying(20) NOT NULL, + slack_user_id character varying(20) NOT NULL, + slack_channel_id character varying(20) NOT NULL, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: slack_state_heap; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.slack_state_heap ( + slack_team character varying(20) NOT NULL, + slack_user_id character varying(20) NOT NULL, + heap_data jsonb NOT NULL, + modified bigint DEFAULT public.now_as_millis(), + created bigint DEFAULT public.now_as_millis(), + active boolean DEFAULT true NOT NULL, + expires bigint DEFAULT 0 +); + + +-- +-- Name: slack_state_stack; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.slack_state_stack ( + slack_team character varying(20) NOT NULL, + slack_user_id character varying(20) NOT NULL, + stack_data jsonb NOT NULL, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: slack_team_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.slack_team_tokens ( + slack_team character varying(20) NOT NULL, + slack_bot_access_token character varying(200) NOT NULL, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: social_settings; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.social_settings ( + uid integer NOT NULL, + polis_pic character varying(3000) +); + + +-- +-- Name: stars; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.stars ( + zid integer NOT NULL, + pid integer NOT NULL, + tid integer NOT NULL, + starred integer NOT NULL, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: suzinvites; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.suzinvites ( + owner integer NOT NULL, + zid integer NOT NULL, + xid text NOT NULL, + created bigint DEFAULT public.now_as_millis(), + suzinvite character varying(32), + uid integer, + modified bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: topic_agenda_selections; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.topic_agenda_selections ( + zid integer NOT NULL, + pid integer NOT NULL, + archetypal_selections jsonb DEFAULT '[]'::jsonb NOT NULL, + delphi_job_id text, + total_selections integer DEFAULT 0 NOT NULL, + created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP +); + + +-- +-- Name: TABLE topic_agenda_selections; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.topic_agenda_selections IS 'Stores user topic agenda selections as archetypal comments that persist across Delphi runs'; + + +-- +-- Name: COLUMN topic_agenda_selections.zid; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.topic_agenda_selections.zid IS 'Conversation ID (foreign key to conversations)'; + + +-- +-- Name: COLUMN topic_agenda_selections.pid; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.topic_agenda_selections.pid IS 'Participant ID (foreign key to participants)'; + + +-- +-- Name: COLUMN topic_agenda_selections.archetypal_selections; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.topic_agenda_selections.archetypal_selections IS 'JSON array of selected topics with their archetypal comments'; + + +-- +-- Name: COLUMN topic_agenda_selections.delphi_job_id; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.topic_agenda_selections.delphi_job_id IS 'ID of the Delphi job that generated the topics'; + + +-- +-- Name: COLUMN topic_agenda_selections.total_selections; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.topic_agenda_selections.total_selections IS 'Total number of topics selected by the user'; + + +-- +-- Name: trashes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.trashes ( + zid integer NOT NULL, + pid integer NOT NULL, + tid integer NOT NULL, + trashed integer NOT NULL, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: treevite_invites; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.treevite_invites ( + id bigint NOT NULL, + zid integer NOT NULL, + wave_id bigint NOT NULL, + parent_invite_id bigint, + invite_code character varying(64) NOT NULL, + status smallint DEFAULT 0 NOT NULL, + invite_owner_pid integer, + invite_used_by_pid integer, + invite_used_at timestamp with time zone, + created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT treevite_invites_status_check CHECK ((status = ANY (ARRAY[0, 1, 2, 3]))) +); + + +-- +-- Name: TABLE treevite_invites; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.treevite_invites IS 'Per-invite records for Treevite, including ownership, usage, and parent-child edges'; + + +-- +-- Name: COLUMN treevite_invites.invite_code; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.treevite_invites.invite_code IS 'Code shared by participants to grant access; unique per conversation'; + + +-- +-- Name: COLUMN treevite_invites.status; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.treevite_invites.status IS '0=unused, 1=used, 2=revoked, 3=expired'; + + +-- +-- Name: COLUMN treevite_invites.invite_owner_pid; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.treevite_invites.invite_owner_pid IS 'PID of participant who owns/distributes this invite (NULL for root invites)'; + + +-- +-- Name: COLUMN treevite_invites.invite_used_by_pid; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.treevite_invites.invite_used_by_pid IS 'PID of participant who consumed the invite (NULL until used)'; + + +-- +-- Name: treevite_invites_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.treevite_invites_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: treevite_invites_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.treevite_invites_id_seq OWNED BY public.treevite_invites.id; + + +-- +-- Name: treevite_login_codes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.treevite_login_codes ( + id bigint NOT NULL, + zid integer NOT NULL, + pid integer NOT NULL, + login_code_hash text NOT NULL, + login_code_fingerprint character varying(128) NOT NULL, + login_code_lookup character varying(128), + fp_kid smallint DEFAULT 1 NOT NULL, + revoked boolean DEFAULT false NOT NULL, + expires_at timestamp with time zone, + last_used_at timestamp with time zone, + created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP +); + + +-- +-- Name: TABLE treevite_login_codes; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.treevite_login_codes IS 'Per-participant Treevite login codes: salted hash for verification plus HMAC fingerprint for lookup'; + + +-- +-- Name: COLUMN treevite_login_codes.login_code_hash; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.treevite_login_codes.login_code_hash IS 'Slow salted hash (argon2/bcrypt) of the participant login code; the raw code is never stored'; + + +-- +-- Name: COLUMN treevite_login_codes.login_code_fingerprint; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.treevite_login_codes.login_code_fingerprint IS 'Indexable HMAC-derived fingerprint scoped by conversation for fast lookup'; + + +-- +-- Name: COLUMN treevite_login_codes.login_code_lookup; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.treevite_login_codes.login_code_lookup IS 'Peppered SHA-256 of login_code for O(1) lookup; verify with bcrypt hash after lookup'; + + +-- +-- Name: treevite_login_codes_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.treevite_login_codes_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: treevite_login_codes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.treevite_login_codes_id_seq OWNED BY public.treevite_login_codes.id; + + +-- +-- Name: treevite_waves; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.treevite_waves ( + id bigint NOT NULL, + zid integer NOT NULL, + wave integer NOT NULL, + parent_wave integer, + size integer, + invites_per_user integer NOT NULL, + owner_invites integer DEFAULT 0 NOT NULL, + created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT treevite_waves_invites_per_user_check CHECK ((invites_per_user >= 0)), + CONSTRAINT treevite_waves_not_both_zero CHECK (((invites_per_user > 0) OR (owner_invites > 0))), + CONSTRAINT treevite_waves_owner_invites_check CHECK ((owner_invites >= 0)), + CONSTRAINT treevite_waves_parent_wave_check CHECK (((parent_wave IS NULL) OR (parent_wave >= 0))), + CONSTRAINT treevite_waves_size_check CHECK (((size IS NULL) OR (size >= 0))), + CONSTRAINT treevite_waves_wave_check CHECK ((wave >= 1)) +); + + +-- +-- Name: TABLE treevite_waves; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON TABLE public.treevite_waves IS 'Per-wave configuration and summary for Treevite invites'; + + +-- +-- Name: COLUMN treevite_waves.wave; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.treevite_waves.wave IS 'Wave number (1-based); unique per conversation'; + + +-- +-- Name: COLUMN treevite_waves.parent_wave; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.treevite_waves.parent_wave IS 'Parent wave number for deriving next wave; 0 for root, NULL means default to greatest existing wave for this zid or 0 if none'; + + +-- +-- Name: COLUMN treevite_waves.size; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.treevite_waves.size IS 'Optional cached size of the wave; derived as (parent_size or 1) * invites_per_user + owner_invites'; + + +-- +-- Name: COLUMN treevite_waves.invites_per_user; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.treevite_waves.invites_per_user IS 'Number of invites granted to each participant in this wave'; + + +-- +-- Name: COLUMN treevite_waves.owner_invites; Type: COMMENT; Schema: public; Owner: - +-- + +COMMENT ON COLUMN public.treevite_waves.owner_invites IS 'Number of owner-controlled invites added to this wave'; + + +-- +-- Name: treevite_waves_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.treevite_waves_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: treevite_waves_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.treevite_waves_id_seq OWNED BY public.treevite_waves.id; + + +-- +-- Name: twitter_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.twitter_users ( + uid integer NOT NULL, + twitter_user_id bigint NOT NULL, + screen_name character varying(999) NOT NULL, + followers_count integer NOT NULL, + friends_count integer NOT NULL, + verified boolean NOT NULL, + profile_image_url_https character varying(9999), + modified bigint DEFAULT public.now_as_millis() NOT NULL, + created bigint DEFAULT public.now_as_millis() NOT NULL, + location character varying(9999), + response json, + name character varying(9999) +); + + +-- +-- Name: upvotes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.upvotes ( + uid integer, + zid integer +); + + +-- +-- Name: users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.users ( + uid integer NOT NULL, + hname character varying(746), + pwhash character varying(128), + username character varying(128), + email character varying(256), + is_owner boolean DEFAULT false, + zinvite character varying(300), + oinvite character varying(300), + created bigint DEFAULT public.now_as_millis(), + tut smallint DEFAULT 0, + site_id character varying(256) DEFAULT public.random_polis_site_id() NOT NULL, + site_owner boolean DEFAULT true, + test boolean DEFAULT false +); + + +-- +-- Name: users_uid_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.users_uid_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: users_uid_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.users_uid_seq OWNED BY public.users.uid; + + +-- +-- Name: votes_latest_unique; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.votes_latest_unique ( + zid integer NOT NULL, + pid integer NOT NULL, + tid integer NOT NULL, + vote smallint, + weight_x_32767 smallint DEFAULT 0, + modified bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: worker_tasks; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.worker_tasks ( + task_data jsonb NOT NULL, + created bigint DEFAULT public.now_as_millis(), + finished_time bigint, + task_type text, + math_env character varying(999) NOT NULL, + task_bucket bigint, + attempts smallint DEFAULT 0 NOT NULL +); + + +-- +-- Name: xid_whitelist; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.xid_whitelist ( + owner integer NOT NULL, + xid text NOT NULL, + created bigint DEFAULT public.now_as_millis() +); + + +-- +-- Name: xids; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.xids ( + uid integer NOT NULL, + owner integer NOT NULL, + xid text NOT NULL, + created bigint DEFAULT public.now_as_millis(), + x_profile_image_url character varying(3000), + x_name character varying(746), + modified bigint DEFAULT public.now_as_millis() NOT NULL, + x_email character varying(256) +); + + +-- +-- Name: zinvites; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.zinvites ( + zid integer NOT NULL, + zinvite character varying(300) NOT NULL, + created bigint DEFAULT public.now_as_millis(), + uuid uuid +); + + +-- +-- Name: contexts context_id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.contexts ALTER COLUMN context_id SET DEFAULT nextval('public.contexts_context_id_seq'::regclass); + + +-- +-- Name: conversations zid; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.conversations ALTER COLUMN zid SET DEFAULT nextval('public.conversations_zid_seq'::regclass); + + +-- +-- Name: courses course_id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.courses ALTER COLUMN course_id SET DEFAULT nextval('public.courses_course_id_seq'::regclass); + + +-- +-- Name: participant_metadata_answers pmaid; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participant_metadata_answers ALTER COLUMN pmaid SET DEFAULT nextval('public.participant_metadata_answers_pmaid_seq'::regclass); + + +-- +-- Name: participant_metadata_questions pmqid; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participant_metadata_questions ALTER COLUMN pmqid SET DEFAULT nextval('public.participant_metadata_questions_pmqid_seq'::regclass); + + +-- +-- Name: reports rid; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.reports ALTER COLUMN rid SET DEFAULT nextval('public.reports_rid_seq'::regclass); + + +-- +-- Name: treevite_invites id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_invites ALTER COLUMN id SET DEFAULT nextval('public.treevite_invites_id_seq'::regclass); + + +-- +-- Name: treevite_login_codes id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_login_codes ALTER COLUMN id SET DEFAULT nextval('public.treevite_login_codes_id_seq'::regclass); + + +-- +-- Name: treevite_waves id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_waves ALTER COLUMN id SET DEFAULT nextval('public.treevite_waves_id_seq'::regclass); + + +-- +-- Name: users uid; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.users ALTER COLUMN uid SET DEFAULT nextval('public.users_uid_seq'::regclass); + + +-- +-- Name: apikeysndvweifu apikeysndvweifu_apikey_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.apikeysndvweifu + ADD CONSTRAINT apikeysndvweifu_apikey_key UNIQUE (apikey); + + +-- +-- Name: auth_tokens auth_tokens_token_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_tokens + ADD CONSTRAINT auth_tokens_token_key UNIQUE (token); + + +-- +-- Name: beta beta_email_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.beta + ADD CONSTRAINT beta_email_key UNIQUE (email); + + +-- +-- Name: comment_translations comment_translations_zid_tid_src_lang_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.comment_translations + ADD CONSTRAINT comment_translations_zid_tid_src_lang_key UNIQUE (zid, tid, src, lang); + + +-- +-- Name: comments comments_tid_unique_constraint; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.comments + ADD CONSTRAINT comments_tid_unique_constraint UNIQUE (zid, tid); + + +-- +-- Name: comments comments_txt_unique_constraint; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.comments + ADD CONSTRAINT comments_txt_unique_constraint UNIQUE (zid, txt); + + +-- +-- Name: conversation_invite_codes conversation_invite_codes_code_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.conversation_invite_codes + ADD CONSTRAINT conversation_invite_codes_code_key UNIQUE (code); + + +-- +-- Name: conversation_subscriptions conversation_subscriptions_zid_uid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.conversation_subscriptions + ADD CONSTRAINT conversation_subscriptions_zid_uid_key UNIQUE (zid, uid); + + +-- +-- Name: conversation_translations conversation_translations_zid_src_lang_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.conversation_translations + ADD CONSTRAINT conversation_translations_zid_src_lang_key UNIQUE (zid, src, lang); + + +-- +-- Name: conversations conversations_zid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.conversations + ADD CONSTRAINT conversations_zid_key UNIQUE (zid); + + +-- +-- Name: courses courses_course_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.courses + ADD CONSTRAINT courses_course_id_key UNIQUE (course_id); + + +-- +-- Name: demographic_data demographic_data_unique_uid; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.demographic_data + ADD CONSTRAINT demographic_data_unique_uid UNIQUE (uid); + + +-- +-- Name: einvites einvites_einvite_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.einvites + ADD CONSTRAINT einvites_einvite_key UNIQUE (einvite); + + +-- +-- Name: users email_unique_check; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.users + ADD CONSTRAINT email_unique_check UNIQUE (email); + + +-- +-- Name: email_validations email_validations_email_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.email_validations + ADD CONSTRAINT email_validations_email_key UNIQUE (email); + + +-- +-- Name: facebook_users facebook_users_fb_user_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.facebook_users + ADD CONSTRAINT facebook_users_fb_user_id_key UNIQUE (fb_user_id); + + +-- +-- Name: facebook_users facebook_users_uid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.facebook_users + ADD CONSTRAINT facebook_users_uid_key UNIQUE (uid); + + +-- +-- Name: jianiuevyew jianiuevyew_uid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.jianiuevyew + ADD CONSTRAINT jianiuevyew_uid_key UNIQUE (uid); + + +-- +-- Name: math_bidtopid math_bidtopid_zid_math_env_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.math_bidtopid + ADD CONSTRAINT math_bidtopid_zid_math_env_key UNIQUE (zid, math_env); + + +-- +-- Name: math_cache math_cache_zid_math_env_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.math_cache + ADD CONSTRAINT math_cache_zid_math_env_key UNIQUE (zid, math_env); + + +-- +-- Name: math_exportstatus math_exportstatus_zid_math_env_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.math_exportstatus + ADD CONSTRAINT math_exportstatus_zid_math_env_key UNIQUE (zid, math_env); + + +-- +-- Name: math_main math_main_zid_math_env_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.math_main + ADD CONSTRAINT math_main_zid_math_env_key UNIQUE (zid, math_env); + + +-- +-- Name: math_profile math_profile_zid_math_env_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.math_profile + ADD CONSTRAINT math_profile_zid_math_env_key UNIQUE (zid, math_env); + + +-- +-- Name: math_ptptstats math_ptptstats_zid_math_env_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.math_ptptstats + ADD CONSTRAINT math_ptptstats_zid_math_env_key UNIQUE (zid, math_env); + + +-- +-- Name: math_report_correlationmatrix math_report_correlationmatrix_rid_math_env_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.math_report_correlationmatrix + ADD CONSTRAINT math_report_correlationmatrix_rid_math_env_key UNIQUE (rid, math_env); + + +-- +-- Name: math_results_dev01 math_results_dev01_zid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.math_results_dev01 + ADD CONSTRAINT math_results_dev01_zid_key UNIQUE (zid); + + +-- +-- Name: math_ticks math_ticks_zid_math_env_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.math_ticks + ADD CONSTRAINT math_ticks_zid_math_env_key UNIQUE (zid, math_env); + + +-- +-- Name: minvites minvites_minvite_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.minvites + ADD CONSTRAINT minvites_minvite_key UNIQUE (minvite); + + +-- +-- Name: moderators moderators_zid_uid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.moderators + ADD CONSTRAINT moderators_zid_uid_key UNIQUE (zid, uid); + + +-- +-- Name: notification_tasks notification_tasks_zid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.notification_tasks + ADD CONSTRAINT notification_tasks_zid_key UNIQUE (zid); + + +-- +-- Name: nyt_users nyt_users_nyt_user_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.nyt_users + ADD CONSTRAINT nyt_users_nyt_user_id_key UNIQUE (nyt_user_id); + + +-- +-- Name: nyt_users nyt_users_uid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.nyt_users + ADD CONSTRAINT nyt_users_uid_key UNIQUE (uid); + + +-- +-- Name: oidc_user_mappings oidc_user_mappings_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.oidc_user_mappings + ADD CONSTRAINT oidc_user_mappings_pkey PRIMARY KEY (oidc_sub); + + +-- +-- Name: oidc_user_mappings oidc_user_mappings_uid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.oidc_user_mappings + ADD CONSTRAINT oidc_user_mappings_uid_key UNIQUE (uid); + + +-- +-- Name: oinvites oinvites_oinvite_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.oinvites + ADD CONSTRAINT oinvites_oinvite_key UNIQUE (oinvite); + + +-- +-- Name: page_ids page_ids_site_id_page_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.page_ids + ADD CONSTRAINT page_ids_site_id_page_id_key UNIQUE (site_id, page_id); + + +-- +-- Name: participant_locations participant_locations_zid_uid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participant_locations + ADD CONSTRAINT participant_locations_zid_uid_key UNIQUE (zid, uid); + + +-- +-- Name: participant_metadata_answers participant_metadata_answers_pmaid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participant_metadata_answers + ADD CONSTRAINT participant_metadata_answers_pmaid_key UNIQUE (pmaid); + + +-- +-- Name: participant_metadata_answers participant_metadata_answers_pmqid_zid_value_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participant_metadata_answers + ADD CONSTRAINT participant_metadata_answers_pmqid_zid_value_key UNIQUE (pmqid, zid, value); + + +-- +-- Name: participant_metadata_choices participant_metadata_choices_zid_pid_pmqid_pmaid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participant_metadata_choices + ADD CONSTRAINT participant_metadata_choices_zid_pid_pmqid_pmaid_key UNIQUE (zid, pid, pmqid, pmaid); + + +-- +-- Name: participant_metadata_questions participant_metadata_questions_pmqid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participant_metadata_questions + ADD CONSTRAINT participant_metadata_questions_pmqid_key UNIQUE (pmqid); + + +-- +-- Name: participant_metadata_questions participant_metadata_questions_zid_key_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participant_metadata_questions + ADD CONSTRAINT participant_metadata_questions_zid_key_key UNIQUE (zid, key); + + +-- +-- Name: participants_extended participants_extended_zid_uid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participants_extended + ADD CONSTRAINT participants_extended_zid_uid_key UNIQUE (zid, uid); + + +-- +-- Name: participants participants_zid_pid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participants + ADD CONSTRAINT participants_zid_pid_key UNIQUE (zid, pid); + + +-- +-- Name: participants participants_zid_uid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participants + ADD CONSTRAINT participants_zid_uid_key UNIQUE (zid, uid); + + +-- +-- Name: permanentcookiezidjoins permanentcookiezidjoins_zid_cookie_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.permanentcookiezidjoins + ADD CONSTRAINT permanentcookiezidjoins_zid_cookie_key UNIQUE (zid, cookie); + + +-- +-- Name: pwreset_tokens pwreset_tokens_token_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.pwreset_tokens + ADD CONSTRAINT pwreset_tokens_token_key UNIQUE (token); + + +-- +-- Name: queue queue_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.queue + ADD CONSTRAINT queue_pkey PRIMARY KEY (itemid); + + +-- +-- Name: report_comment_selections report_comment_selections_rid_tid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.report_comment_selections + ADD CONSTRAINT report_comment_selections_rid_tid_key UNIQUE (rid, tid); + + +-- +-- Name: reports reports_report_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.reports + ADD CONSTRAINT reports_report_id_key UNIQUE (report_id); + + +-- +-- Name: reports reports_rid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.reports + ADD CONSTRAINT reports_rid_key UNIQUE (rid); + + +-- +-- Name: slack_participants_waiting_for_comments slack_participants_waiting_for_slack_team_slack_user_id_zid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.slack_participants_waiting_for_comments + ADD CONSTRAINT slack_participants_waiting_for_slack_team_slack_user_id_zid_key UNIQUE (slack_team, slack_user_id, zid); + + +-- +-- Name: slack_state_heap slack_state_heap_slack_team_slack_user_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.slack_state_heap + ADD CONSTRAINT slack_state_heap_slack_team_slack_user_id_key UNIQUE (slack_team, slack_user_id); + + +-- +-- Name: slack_state_stack slack_state_stack_slack_team_slack_user_id_created_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.slack_state_stack + ADD CONSTRAINT slack_state_stack_slack_team_slack_user_id_created_key UNIQUE (slack_team, slack_user_id, created); + + +-- +-- Name: slack_team_tokens slack_team_tokens_slack_team_slack_bot_access_token_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.slack_team_tokens + ADD CONSTRAINT slack_team_tokens_slack_team_slack_bot_access_token_key UNIQUE (slack_team, slack_bot_access_token); + + +-- +-- Name: social_settings social_settings_uid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.social_settings + ADD CONSTRAINT social_settings_uid_key UNIQUE (uid); + + +-- +-- Name: suzinvites suzinvites_suzinvite_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.suzinvites + ADD CONSTRAINT suzinvites_suzinvite_key UNIQUE (suzinvite); + + +-- +-- Name: topic_agenda_selections topic_agenda_selections_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.topic_agenda_selections + ADD CONSTRAINT topic_agenda_selections_pkey PRIMARY KEY (zid, pid); + + +-- +-- Name: treevite_invites treevite_invites_code_unique; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_invites + ADD CONSTRAINT treevite_invites_code_unique UNIQUE (zid, invite_code); + + +-- +-- Name: treevite_invites treevite_invites_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_invites + ADD CONSTRAINT treevite_invites_pkey PRIMARY KEY (id); + + +-- +-- Name: treevite_login_codes treevite_login_codes_fp_unique; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_login_codes + ADD CONSTRAINT treevite_login_codes_fp_unique UNIQUE (zid, login_code_fingerprint); + + +-- +-- Name: treevite_login_codes treevite_login_codes_lookup_unique; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_login_codes + ADD CONSTRAINT treevite_login_codes_lookup_unique UNIQUE (zid, login_code_lookup); + + +-- +-- Name: treevite_login_codes treevite_login_codes_pid_unique; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_login_codes + ADD CONSTRAINT treevite_login_codes_pid_unique UNIQUE (zid, pid); + + +-- +-- Name: treevite_login_codes treevite_login_codes_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_login_codes + ADD CONSTRAINT treevite_login_codes_pkey PRIMARY KEY (id); + + +-- +-- Name: treevite_waves treevite_waves_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_waves + ADD CONSTRAINT treevite_waves_pkey PRIMARY KEY (id); + + +-- +-- Name: treevite_waves treevite_waves_zid_wave_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_waves + ADD CONSTRAINT treevite_waves_zid_wave_key UNIQUE (zid, wave); + + +-- +-- Name: twitter_users twitter_users_twitter_user_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.twitter_users + ADD CONSTRAINT twitter_users_twitter_user_id_key UNIQUE (twitter_user_id); + + +-- +-- Name: twitter_users twitter_users_uid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.twitter_users + ADD CONSTRAINT twitter_users_uid_key UNIQUE (uid); + + +-- +-- Name: upvotes upvotes_uid_zid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.upvotes + ADD CONSTRAINT upvotes_uid_zid_key UNIQUE (uid, zid); + + +-- +-- Name: users users_uid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.users + ADD CONSTRAINT users_uid_key UNIQUE (uid); + + +-- +-- Name: votes_latest_unique votes_latest_unique_zid_pid_tid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.votes_latest_unique + ADD CONSTRAINT votes_latest_unique_zid_pid_tid_key UNIQUE (zid, pid, tid); + + +-- +-- Name: xid_whitelist xid_whitelist_owner_xid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.xid_whitelist + ADD CONSTRAINT xid_whitelist_owner_xid_key UNIQUE (owner, xid); + + +-- +-- Name: xids xids_owner_xid_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.xids + ADD CONSTRAINT xids_owner_xid_key UNIQUE (owner, xid); + + +-- +-- Name: xids xids_unique_owner_uid_constraint; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.xids + ADD CONSTRAINT xids_unique_owner_uid_constraint UNIQUE (owner, uid); + + +-- +-- Name: zinvites zinvites_zinvite_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.zinvites + ADD CONSTRAINT zinvites_zinvite_key UNIQUE (zinvite); + + +-- +-- Name: apikeysndvweifu_apikey_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX apikeysndvweifu_apikey_idx ON public.apikeysndvweifu USING btree (apikey); + + +-- +-- Name: apikeysndvweifu_uid_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX apikeysndvweifu_uid_idx ON public.apikeysndvweifu USING btree (uid); + + +-- +-- Name: auth_tokens_token_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX auth_tokens_token_idx ON public.auth_tokens USING btree (token); + + +-- +-- Name: comment_translations_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX comment_translations_idx ON public.comment_translations USING btree (zid, tid); + + +-- +-- Name: comments_zid_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX comments_zid_idx ON public.comments USING btree (zid); + + +-- +-- Name: conversation_translations_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX conversation_translations_idx ON public.conversation_translations USING btree (zid); + + +-- +-- Name: conversations_owner_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX conversations_owner_idx ON public.conversations USING btree (owner); + + +-- +-- Name: conversations_zid_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX conversations_zid_index ON public.conversations USING btree (zid); + + +-- +-- Name: course_id_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX course_id_idx ON public.courses USING btree (course_id); + + +-- +-- Name: idx_oidc_mappings_uid; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_oidc_mappings_uid ON public.oidc_user_mappings USING btree (uid); + + +-- +-- Name: idx_topic_agenda_selections_created_at; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_topic_agenda_selections_created_at ON public.topic_agenda_selections USING btree (created_at); + + +-- +-- Name: idx_topic_agenda_selections_delphi_job_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_topic_agenda_selections_delphi_job_id ON public.topic_agenda_selections USING btree (delphi_job_id); + + +-- +-- Name: idx_topic_agenda_selections_pid; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_topic_agenda_selections_pid ON public.topic_agenda_selections USING btree (pid); + + +-- +-- Name: idx_topic_agenda_selections_zid; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_topic_agenda_selections_zid ON public.topic_agenda_selections USING btree (zid); + + +-- +-- Name: idx_treevite_invites_code; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_treevite_invites_code ON public.treevite_invites USING btree (invite_code); + + +-- +-- Name: idx_treevite_invites_owner_pid; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_treevite_invites_owner_pid ON public.treevite_invites USING btree (invite_owner_pid); + + +-- +-- Name: idx_treevite_invites_parent; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_treevite_invites_parent ON public.treevite_invites USING btree (parent_invite_id); + + +-- +-- Name: idx_treevite_invites_used_by_pid; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_treevite_invites_used_by_pid ON public.treevite_invites USING btree (invite_used_by_pid); + + +-- +-- Name: idx_treevite_invites_wave_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_treevite_invites_wave_id ON public.treevite_invites USING btree (wave_id); + + +-- +-- Name: idx_treevite_invites_zid; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_treevite_invites_zid ON public.treevite_invites USING btree (zid); + + +-- +-- Name: idx_treevite_invites_zid_status; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_treevite_invites_zid_status ON public.treevite_invites USING btree (zid, status); + + +-- +-- Name: idx_treevite_login_codes_fp; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_treevite_login_codes_fp ON public.treevite_login_codes USING btree (login_code_fingerprint); + + +-- +-- Name: idx_treevite_login_codes_lookup; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_treevite_login_codes_lookup ON public.treevite_login_codes USING btree (zid, login_code_lookup); + + +-- +-- Name: idx_treevite_login_codes_pid; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_treevite_login_codes_pid ON public.treevite_login_codes USING btree (pid); + + +-- +-- Name: idx_treevite_login_codes_zid; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_treevite_login_codes_zid ON public.treevite_login_codes USING btree (zid); + + +-- +-- Name: idx_treevite_waves_parent; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_treevite_waves_parent ON public.treevite_waves USING btree (zid, parent_wave); + + +-- +-- Name: idx_treevite_waves_wave; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_treevite_waves_wave ON public.treevite_waves USING btree (wave); + + +-- +-- Name: idx_treevite_waves_zid; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_treevite_waves_zid ON public.treevite_waves USING btree (zid); + + +-- +-- Name: main_main_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX main_main_idx ON public.math_main USING btree (zid); + + +-- +-- Name: main_profile_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX main_profile_idx ON public.math_profile USING btree (zid); + + +-- +-- Name: math_bidtopid_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX math_bidtopid_idx ON public.math_bidtopid USING btree (zid); + + +-- +-- Name: math_cache_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX math_cache_idx ON public.math_cache USING btree (zid); + + +-- +-- Name: math_exportstatus_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX math_exportstatus_idx ON public.math_exportstatus USING btree (zid); + + +-- +-- Name: math_math_report_correlationmatrix_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX math_math_report_correlationmatrix_idx ON public.math_report_correlationmatrix USING btree (rid); + + +-- +-- Name: math_ptptstats_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX math_ptptstats_idx ON public.math_ptptstats USING btree (zid); + + +-- +-- Name: participants_conv_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX participants_conv_idx ON public.participants USING btree (zid); + + +-- +-- Name: participants_conv_uid_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX participants_conv_uid_idx ON public.participants USING btree (uid); + + +-- +-- Name: participants_uid_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX participants_uid_index ON public.participants USING btree (uid); + + +-- +-- Name: pwreset_tokens_token_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX pwreset_tokens_token_idx ON public.pwreset_tokens USING btree (token); + + +-- +-- Name: site_domain_whitelist_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX site_domain_whitelist_idx ON public.users USING btree (site_id); + + +-- +-- Name: suzinvites_owner_zid_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX suzinvites_owner_zid_idx ON public.suzinvites USING btree (owner, zid); + + +-- +-- Name: votes_latest_unique_zid_tid_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX votes_latest_unique_zid_tid_idx ON public.votes USING btree (zid, tid); + + +-- +-- Name: votes_zid_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX votes_zid_idx ON public.votes USING btree (zid); + + +-- +-- Name: votes_zid_pid_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX votes_zid_pid_idx ON public.votes USING btree (zid, pid); + + +-- +-- Name: xid_whitelist_owner_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX xid_whitelist_owner_idx ON public.xid_whitelist USING btree (owner); + + +-- +-- Name: xids_owner_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX xids_owner_idx ON public.xids USING btree (owner); + + +-- +-- Name: zinvites_zid_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX zinvites_zid_idx ON public.zinvites USING btree (zid); + + +-- +-- Name: votes on_vote_insert_update_unique_table; Type: RULE; Schema: public; Owner: - +-- + +CREATE RULE on_vote_insert_update_unique_table AS + ON INSERT TO public.votes DO INSERT INTO public.votes_latest_unique (zid, pid, tid, vote, weight_x_32767, modified) + VALUES (new.zid, new.pid, new.tid, new.vote, new.weight_x_32767, new.created) ON CONFLICT(zid, pid, tid) DO UPDATE SET vote = excluded.vote, modified = excluded.modified; + + +-- +-- Name: participants pid_auto; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER pid_auto BEFORE INSERT ON public.participants FOR EACH ROW EXECUTE FUNCTION public.pid_auto(); + + +-- +-- Name: participants pid_auto_unlock; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER pid_auto_unlock AFTER INSERT ON public.participants FOR EACH ROW EXECUTE FUNCTION public.pid_auto_unlock(); + + +-- +-- Name: comments tid_auto; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER tid_auto BEFORE INSERT ON public.comments FOR EACH ROW EXECUTE FUNCTION public.tid_auto(); + + +-- +-- Name: comments tid_auto_unlock; Type: TRIGGER; Schema: public; Owner: - +-- + +CREATE TRIGGER tid_auto_unlock AFTER INSERT ON public.comments FOR EACH ROW EXECUTE FUNCTION public.tid_auto_unlock(); + + +-- +-- Name: apikeysndvweifu apikeysndvweifu_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.apikeysndvweifu + ADD CONSTRAINT apikeysndvweifu_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: auth_tokens auth_tokens_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.auth_tokens + ADD CONSTRAINT auth_tokens_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: comment_translations comment_translations_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.comment_translations + ADD CONSTRAINT comment_translations_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: comments comments_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.comments + ADD CONSTRAINT comments_zid_fkey FOREIGN KEY (zid, pid) REFERENCES public.participants(zid, pid); + + +-- +-- Name: contexts contexts_creator_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.contexts + ADD CONSTRAINT contexts_creator_fkey FOREIGN KEY (creator) REFERENCES public.users(uid); + + +-- +-- Name: contributor_agreement_signatures contributer_agreement_signatures_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.contributor_agreement_signatures + ADD CONSTRAINT contributer_agreement_signatures_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: conversation_invite_codes conversation_invite_codes_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.conversation_invite_codes + ADD CONSTRAINT conversation_invite_codes_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: conversation_subscriptions conversation_subscriptions_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.conversation_subscriptions + ADD CONSTRAINT conversation_subscriptions_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: conversation_subscriptions conversation_subscriptions_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.conversation_subscriptions + ADD CONSTRAINT conversation_subscriptions_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: conversation_translations conversation_translations_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.conversation_translations + ADD CONSTRAINT conversation_translations_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: conversations conversations_actual_ownr_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.conversations + ADD CONSTRAINT conversations_actual_ownr_fkey FOREIGN KEY (org_id) REFERENCES public.users(uid); + + +-- +-- Name: conversations conversations_course_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.conversations + ADD CONSTRAINT conversations_course_id_fkey FOREIGN KEY (course_id) REFERENCES public.courses(course_id); + + +-- +-- Name: conversations conversations_owner_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.conversations + ADD CONSTRAINT conversations_owner_fkey FOREIGN KEY (owner) REFERENCES public.users(uid); + + +-- +-- Name: courses courses_owner_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.courses + ADD CONSTRAINT courses_owner_fkey FOREIGN KEY (owner) REFERENCES public.users(uid); + + +-- +-- Name: demographic_data demographic_data_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.demographic_data + ADD CONSTRAINT demographic_data_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: error_reports error_reports_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.error_reports + ADD CONSTRAINT error_reports_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: error_reports error_reports_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.error_reports + ADD CONSTRAINT error_reports_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: facebook_friends facebook_friends_friend_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.facebook_friends + ADD CONSTRAINT facebook_friends_friend_fkey FOREIGN KEY (friend) REFERENCES public.users(uid); + + +-- +-- Name: facebook_friends facebook_friends_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.facebook_friends + ADD CONSTRAINT facebook_friends_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: facebook_users facebook_users_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.facebook_users + ADD CONSTRAINT facebook_users_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: topic_agenda_selections fk_conversation; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.topic_agenda_selections + ADD CONSTRAINT fk_conversation FOREIGN KEY (zid) REFERENCES public.conversations(zid) ON DELETE CASCADE; + + +-- +-- Name: topic_agenda_selections fk_participant; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.topic_agenda_selections + ADD CONSTRAINT fk_participant FOREIGN KEY (zid, pid) REFERENCES public.participants(zid, pid) ON DELETE CASCADE; + + +-- +-- Name: inviters inviters_inviter_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.inviters + ADD CONSTRAINT inviters_inviter_uid_fkey FOREIGN KEY (inviter_uid) REFERENCES public.users(uid); + + +-- +-- Name: jianiuevyew jianiuevyew_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.jianiuevyew + ADD CONSTRAINT jianiuevyew_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: math_bidtopid math_bidtopid_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.math_bidtopid + ADD CONSTRAINT math_bidtopid_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: math_cache math_cache_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.math_cache + ADD CONSTRAINT math_cache_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: math_exportstatus math_exportstatus_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.math_exportstatus + ADD CONSTRAINT math_exportstatus_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: math_main math_main_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.math_main + ADD CONSTRAINT math_main_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: math_profile math_profile_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.math_profile + ADD CONSTRAINT math_profile_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: math_ptptstats math_ptptstats_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.math_ptptstats + ADD CONSTRAINT math_ptptstats_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: math_report_correlationmatrix math_report_correlationmatrix_rid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.math_report_correlationmatrix + ADD CONSTRAINT math_report_correlationmatrix_rid_fkey FOREIGN KEY (rid) REFERENCES public.reports(rid); + + +-- +-- Name: math_ticks math_ticks_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.math_ticks + ADD CONSTRAINT math_ticks_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: metrics metrics_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.metrics + ADD CONSTRAINT metrics_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: minvites minvites_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.minvites + ADD CONSTRAINT minvites_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: moderators moderators_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.moderators + ADD CONSTRAINT moderators_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: moderators moderators_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.moderators + ADD CONSTRAINT moderators_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: notification_tasks notification_tasks_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.notification_tasks + ADD CONSTRAINT notification_tasks_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: nyt_users nyt_users_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.nyt_users + ADD CONSTRAINT nyt_users_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: oidc_user_mappings oidc_user_mappings_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.oidc_user_mappings + ADD CONSTRAINT oidc_user_mappings_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid) ON DELETE CASCADE; + + +-- +-- Name: page_ids page_ids_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.page_ids + ADD CONSTRAINT page_ids_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: participant_locations participant_locations_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participant_locations + ADD CONSTRAINT participant_locations_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: participant_locations participant_locations_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participant_locations + ADD CONSTRAINT participant_locations_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: participant_metadata_answers participant_metadata_answers_pmqid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participant_metadata_answers + ADD CONSTRAINT participant_metadata_answers_pmqid_fkey FOREIGN KEY (pmqid) REFERENCES public.participant_metadata_questions(pmqid); + + +-- +-- Name: participant_metadata_answers participant_metadata_answers_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participant_metadata_answers + ADD CONSTRAINT participant_metadata_answers_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: participant_metadata_choices participant_metadata_choices_pmaid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participant_metadata_choices + ADD CONSTRAINT participant_metadata_choices_pmaid_fkey FOREIGN KEY (pmaid) REFERENCES public.participant_metadata_answers(pmaid); + + +-- +-- Name: participant_metadata_choices participant_metadata_choices_pmqid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participant_metadata_choices + ADD CONSTRAINT participant_metadata_choices_pmqid_fkey FOREIGN KEY (pmqid) REFERENCES public.participant_metadata_questions(pmqid); + + +-- +-- Name: participant_metadata_choices participant_metadata_choices_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participant_metadata_choices + ADD CONSTRAINT participant_metadata_choices_zid_fkey FOREIGN KEY (zid, pid) REFERENCES public.participants(zid, pid); + + +-- +-- Name: participant_metadata_questions participant_metadata_questions_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participant_metadata_questions + ADD CONSTRAINT participant_metadata_questions_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: participants_extended participants_extended_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participants_extended + ADD CONSTRAINT participants_extended_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: participants_extended participants_extended_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participants_extended + ADD CONSTRAINT participants_extended_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: participants participants_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participants + ADD CONSTRAINT participants_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: participants participants_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.participants + ADD CONSTRAINT participants_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: pwreset_tokens pwreset_tokens_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.pwreset_tokens + ADD CONSTRAINT pwreset_tokens_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: report_comment_selections report_comment_selections_rid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.report_comment_selections + ADD CONSTRAINT report_comment_selections_rid_fkey FOREIGN KEY (rid) REFERENCES public.reports(rid); + + +-- +-- Name: report_comment_selections report_comment_selections_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.report_comment_selections + ADD CONSTRAINT report_comment_selections_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: reports reports_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.reports + ADD CONSTRAINT reports_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: slack_participants_waiting_for_comments slack_participants_waiting_for_comments_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.slack_participants_waiting_for_comments + ADD CONSTRAINT slack_participants_waiting_for_comments_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: slack_participants_waiting_for_comments slack_participants_waiting_for_comments_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.slack_participants_waiting_for_comments + ADD CONSTRAINT slack_participants_waiting_for_comments_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: social_settings social_settings_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.social_settings + ADD CONSTRAINT social_settings_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: suzinvites suzinvites_owner_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.suzinvites + ADD CONSTRAINT suzinvites_owner_fkey FOREIGN KEY (owner) REFERENCES public.users(uid); + + +-- +-- Name: suzinvites suzinvites_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.suzinvites + ADD CONSTRAINT suzinvites_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: suzinvites suzinvites_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.suzinvites + ADD CONSTRAINT suzinvites_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: treevite_invites treevite_invites_owner_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_invites + ADD CONSTRAINT treevite_invites_owner_fkey FOREIGN KEY (zid, invite_owner_pid) REFERENCES public.participants(zid, pid); + + +-- +-- Name: treevite_invites treevite_invites_parent_invite_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_invites + ADD CONSTRAINT treevite_invites_parent_invite_id_fkey FOREIGN KEY (parent_invite_id) REFERENCES public.treevite_invites(id) ON DELETE SET NULL; + + +-- +-- Name: treevite_invites treevite_invites_used_by_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_invites + ADD CONSTRAINT treevite_invites_used_by_fkey FOREIGN KEY (zid, invite_used_by_pid) REFERENCES public.participants(zid, pid); + + +-- +-- Name: treevite_invites treevite_invites_wave_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_invites + ADD CONSTRAINT treevite_invites_wave_id_fkey FOREIGN KEY (wave_id) REFERENCES public.treevite_waves(id) ON DELETE CASCADE; + + +-- +-- Name: treevite_invites treevite_invites_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_invites + ADD CONSTRAINT treevite_invites_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid) ON DELETE CASCADE; + + +-- +-- Name: treevite_login_codes treevite_login_codes_participant_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_login_codes + ADD CONSTRAINT treevite_login_codes_participant_fkey FOREIGN KEY (zid, pid) REFERENCES public.participants(zid, pid) ON DELETE CASCADE; + + +-- +-- Name: treevite_waves treevite_waves_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.treevite_waves + ADD CONSTRAINT treevite_waves_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid) ON DELETE CASCADE; + + +-- +-- Name: twitter_users twitter_users_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.twitter_users + ADD CONSTRAINT twitter_users_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: upvotes upvotes_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.upvotes + ADD CONSTRAINT upvotes_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: upvotes upvotes_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.upvotes + ADD CONSTRAINT upvotes_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- Name: xid_whitelist xid_whitelist_owner_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.xid_whitelist + ADD CONSTRAINT xid_whitelist_owner_fkey FOREIGN KEY (owner) REFERENCES public.users(uid); + + +-- +-- Name: xids xids_owner_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.xids + ADD CONSTRAINT xids_owner_fkey FOREIGN KEY (owner) REFERENCES public.users(uid); + + +-- +-- Name: xids xids_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.xids + ADD CONSTRAINT xids_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); + + +-- +-- Name: zinvites zinvites_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.zinvites + ADD CONSTRAINT zinvites_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); + + +-- +-- PostgreSQL database dump complete +-- + +\unrestrict yi3q8cVeyPYXP7xpzDDwpdv2ts6MbtdVFjxHM5kY0sd3iTqn6dEoZ2y0ASHXmgp + diff --git a/delphi/umap_narrative/801_narrative_report_batch.py b/delphi/umap_narrative/801_narrative_report_batch.py index 90c89d4957..1d6a35b7fd 100755 --- a/delphi/umap_narrative/801_narrative_report_batch.py +++ b/delphi/umap_narrative/801_narrative_report_batch.py @@ -19,38 +19,32 @@ --layers: Specific layer numbers to process (e.g., --layers 0 1 2). If not specified, all layers will be processed. """ -import os -import sys -import json -import time -import uuid -import logging import argparse -import boto3 import asyncio -import numpy as np -import pandas as pd +import json +import logging +import os import re # Added re import for regex operations -import requests # Added for HTTP error handling +import sys +import time +import traceback # Added for detailed error tracing +import xml.etree.ElementTree as ET +from collections import defaultdict from datetime import datetime from pathlib import Path -from typing import List, Dict, Any, Optional, Union, Tuple -import xml.etree.ElementTree as ET from xml.dom.minidom import parseString -import csv -import io + +import boto3 import xmltodict -from collections import defaultdict -import traceback # Added for detailed error tracing # Import the model provider sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from umap_narrative.llm_factory_constructor import get_model_provider -from umap_narrative.llm_factory_constructor.model_provider import AnthropicProvider +from polismath_commentgraph.utils.group_data import GroupDataProcessor # Import from local modules -from polismath_commentgraph.utils.storage import PostgresClient, DynamoDBStorage -from polismath_commentgraph.utils.group_data import GroupDataProcessor +from polismath_commentgraph.utils.storage import PostgresClient + +from umap_narrative.llm_factory_constructor import get_model_provider # Configure logging logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') @@ -253,13 +247,11 @@ def _get_math_main_data(self, conversation_id): LIMIT 1 """ - # Use 'prod' as the default math_env (matches the server behavior) - math_env = os.environ.get('MATH_ENV', 'prod') - - results = self.postgres_client.query(sql, {"zid": conversation_id, "math_env": math_env}) + # Use 'prod' as the hardcoded math_env value + results = self.postgres_client.query(sql, {"zid": conversation_id, "math_env": "prod"}) if not results: - logger.warning(f"No math_main data found for conversation {conversation_id} with math_env {math_env}") + logger.warning(f"No math_main data found for conversation {conversation_id} with math_env prod") return None # Parse the JSON data @@ -1209,7 +1201,13 @@ async def submit_batch(self): # Import Anthropic SDK logger.info("Importing Anthropic SDK...") try: - from anthropic import Anthropic, APIError, APIConnectionError, APIResponseValidationError, APIStatusError + from anthropic import ( + Anthropic, + APIConnectionError, + APIError, + APIResponseValidationError, + APIStatusError, + ) logger.info("Successfully imported Anthropic SDK") except ImportError as e: logger.error(f"Failed to import Anthropic SDK: {str(e)}") @@ -1218,7 +1216,13 @@ async def submit_batch(self): try: import subprocess subprocess.check_call([sys.executable, "-m", "pip", "install", "anthropic"]) - from anthropic import Anthropic, APIError, APIConnectionError, APIResponseValidationError, APIStatusError + from anthropic import ( + Anthropic, + APIConnectionError, + APIError, + APIResponseValidationError, + APIStatusError, + ) logger.info("Successfully installed and imported Anthropic SDK") except Exception as e: logger.error(f"Failed to install Anthropic SDK: {str(e)}") @@ -1327,7 +1331,7 @@ async def submit_batch(self): content_item['text'] = content_item['text'][:100] + "... [content truncated for log]" logger.info(f"Sample batch request structure: {json.dumps(debug_request, indent=2)}") - logger.info(f"Using format that matches working example from other project") + logger.info("Using format that matches working example from other project") except Exception as e: logger.error(f"Error formatting batch requests: {str(e)}") @@ -1414,15 +1418,15 @@ async def submit_batch(self): if 'batch_id' in job_item: logger.info(f"VERIFICATION SUCCESS: batch_id found in job record: {job_item['batch_id']}") else: - logger.error(f"VERIFICATION FAILED: batch_id not found in job record!") + logger.error("VERIFICATION FAILED: batch_id not found in job record!") logger.error(f"Job fields: {list(job_item.keys())}") else: - logger.error(f"Could not verify update - job not found!") + logger.error("Could not verify update - job not found!") logger.info(f"Successfully updated job {self.job_id} with batch information") logger.info(f"Batch ID: {batch.id} stored in job record") logger.info(f"DynamoDB update response: {update_response}") - logger.info(f"Job is now in PROCESSING state - poller will run batch status checks") + logger.info("Job is now in PROCESSING state - poller will run batch status checks") # Schedule a batch status check job to run in 60 seconds try: @@ -1518,14 +1522,14 @@ async def main(): os.environ.setdefault('DATABASE_PASSWORD', '') # Print database connection info - logger.info(f"Database connection info:") + logger.info("Database connection info:") logger.info(f"- HOST: {os.environ.get('DATABASE_HOST')}") logger.info(f"- PORT: {os.environ.get('DATABASE_PORT')}") logger.info(f"- DATABASE: {os.environ.get('DATABASE_NAME')}") logger.info(f"- USER: {os.environ.get('DATABASE_USER')}") # Print execution summary - logger.info(f"Running narrative report generator with the following settings:") + logger.info("Running narrative report generator with the following settings:") logger.info(f"- Conversation ID: {args.conversation_id}") logger.info(f"- Model: {args.model}") logger.info(f"- Cache: {'disabled' if args.no_cache else 'enabled'}") @@ -1533,7 +1537,7 @@ async def main(): if args.layers: logger.info(f"- Layers to process: {args.layers}") else: - logger.info(f"- Layers to process: all available layers") + logger.info("- Layers to process: all available layers") if job_id: logger.info(f"- Job ID: {job_id}") if report_id: @@ -1554,15 +1558,15 @@ async def main(): result = await generator.submit_batch() if result: - logger.info(f"Narrative reports generated successfully") - print(f"Narrative reports generated successfully") + logger.info("Narrative reports generated successfully") + print("Narrative reports generated successfully") if job_id: print(f"Job ID: {job_id}") if report_id: print(f"Reports stored for report_id: {report_id}") else: - logger.error(f"Failed to generate narrative reports") - print(f"Failed to generate narrative reports. See logs for details.") + logger.error("Failed to generate narrative reports") + print("Failed to generate narrative reports. See logs for details.") # Exit with error code sys.exit(1) From 2bd097278bc00cd3cb8deff7ab46eab1862921f7 Mon Sep 17 00:00:00 2001 From: Bennie Rosas Date: Sat, 25 Oct 2025 05:01:18 -0500 Subject: [PATCH 4/7] math: remove MATH_ENV; cleanup; remove dead code --- math/.vscode/settings.json | 10 + math/bin/util-server | 7 - math/deps.edn | 55 +- math/dev.env.sh | 1 - math/dev/user.clj | 403 +++---- math/doc/commands.md | 6 +- math/doc/configuration.md | 22 +- math/doc/update.md | 4 +- math/example.env | 3 +- math/perf | 324 ------ math/python_conversion/.gitignore | 54 - math/resources/config.edn | 22 - math/src/data_readers.clj | 7 - math/src/polismath/components/config.clj | 103 +- .../polismath/components/core_matrix_boot.clj | 17 +- math/src/polismath/components/env.clj | 14 - math/src/polismath/components/logger.clj | 35 +- math/src/polismath/components/postgres.clj | 239 ++--- math/src/polismath/components/random.clj | 7 - math/src/polismath/components/server.clj | 72 -- math/src/polismath/conv_man.clj | 118 +-- math/src/polismath/darwin/core.clj | 198 +--- math/src/polismath/darwin/export.clj | 412 ++------ math/src/polismath/math/clusters.clj | 257 ++--- math/src/polismath/math/conversation.clj | 986 +++++++++--------- math/src/polismath/math/corr.clj | 247 ----- math/src/polismath/math/named_matrix.clj | 189 ++-- math/src/polismath/math/pca.clj | 95 +- math/src/polismath/math/repness.clj | 249 +++-- math/src/polismath/math/stats.clj | 22 +- math/src/polismath/meta/metrics.clj | 26 +- math/src/polismath/meta/microscope.clj | 68 +- math/src/polismath/meta/notify.clj | 45 +- math/src/polismath/meta/simulation.clj | 270 ----- math/src/polismath/poller.clj | 14 +- math/src/polismath/runner.clj | 144 +-- math/src/polismath/stormspec.clj | 142 --- math/src/polismath/system.clj | 59 +- math/src/polismath/tasks.clj | 22 +- math/src/polismath/util/pretty_printers.clj | 29 - math/src/polismath/utils.clj | 108 +- math/test/conv_man_tests.clj | 10 +- math/tmp/.touchfile | 0 43 files changed, 1490 insertions(+), 3625 deletions(-) create mode 100644 math/.vscode/settings.json delete mode 100755 math/bin/util-server delete mode 100644 math/perf delete mode 100644 math/python_conversion/.gitignore delete mode 100644 math/src/data_readers.clj delete mode 100644 math/src/polismath/components/env.clj delete mode 100644 math/src/polismath/components/random.clj delete mode 100644 math/src/polismath/components/server.clj delete mode 100644 math/src/polismath/math/corr.clj delete mode 100644 math/src/polismath/meta/simulation.clj delete mode 100644 math/src/polismath/stormspec.clj delete mode 100644 math/src/polismath/util/pretty_printers.clj delete mode 100644 math/tmp/.touchfile diff --git a/math/.vscode/settings.json b/math/.vscode/settings.json new file mode 100644 index 0000000000..868a17abb6 --- /dev/null +++ b/math/.vscode/settings.json @@ -0,0 +1,10 @@ +{ + "[clojure]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "betterthantomorrow.calva", + "editor.codeActionsOnSave": { + "source.fixAll.eslint": "explicit", + "source.organizeImports": "explicit" + } + } +} diff --git a/math/bin/util-server b/math/bin/util-server deleted file mode 100755 index 7fdd45c538..0000000000 --- a/math/bin/util-server +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -# Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . - -while [ 1 ] -do - lein with-profile production trampoline run -m polismath.server -done diff --git a/math/deps.edn b/math/deps.edn index e040070755..2a484c4833 100644 --- a/math/deps.edn +++ b/math/deps.edn @@ -1,54 +1,36 @@ {:paths ["src"] - :deps {org.clojure/clojure {:mvn/version "1.11.1"} - org.clojure/spec.alpha {:mvn/version "0.3.218"} - org.clojure/core.async {:mvn/version "1.6.673"} - org.clojure/data.csv {:mvn/version "1.0.1"} - org.clojure/math.numeric-tower {:mvn/version "0.0.5"} - org.clojure/core.match {:mvn/version "1.0.1"} - org.clojure/tools.namespace {:mvn/version "1.3.0"} - org.clojure/tools.logging {:mvn/version "1.2.4"} - org.clojure/tools.trace {:mvn/version "0.7.11"} - org.clojure/tools.reader {:mvn/version "1.3.6"} + :deps {org.clojure/clojure {:mvn/version "1.11.4"} + org.clojure/spec.alpha {:mvn/version "0.5.238"} + org.clojure/core.async {:mvn/version "1.6.681"} + org.clojure/data.csv {:mvn/version "1.1.0"} + org.clojure/math.numeric-tower {:mvn/version "0.1.0"} + org.clojure/core.match {:mvn/version "1.1.0"} + org.clojure/tools.namespace {:mvn/version "1.5.0"} + org.clojure/tools.logging {:mvn/version "1.3.0"} + org.clojure/tools.trace {:mvn/version "0.8.0"} + org.clojure/tools.reader {:mvn/version "1.5.0"} - org.flatland/ordered {:mvn/version "1.15.10"} + org.flatland/ordered {:mvn/version "1.15.12"} ;; Other stuff commons-collections/commons-collections {:mvn/version "20040616"} - cheshire/cheshire {:mvn/version "5.11.0"} + cheshire/cheshire {:mvn/version "5.13.0"} com.taoensso/timbre {:mvn/version "5.2.1"} - ;; Troublesome carmine... was using this for simulation stuff - ;[com.taoensso/carmine "2.7.0" :exclusions [org.clojure/clojure]] ;; Updates; requires fixing index conflict between named-matrix and core.matrix net.mikera/core.matrix {:mvn/version "0.63.0"} net.mikera/vectorz-clj {:mvn/version "0.48.0"} - ;[net.mikera/core.matrix "0.23.0"] net.mikera/core.matrix.stats {:mvn/version "0.7.0"} criterium/criterium {:mvn/version "0.4.6"} - clj-http/clj-http {:mvn/version "3.12.3"} - ;; We should be able to switch back to this now that we aren't using storm org.clojure/tools.cli {:mvn/version "1.0.214"} - ;; implicitly requires jetty, component and ring + com.stuartsierra/component {:mvn/version "1.1.0"} ring/ring-core {:mvn/version "1.9.6" :exclusions [clj-time/clj-time]} - ring-jetty-component/ring-jetty-component {:mvn/version "0.3.1" :exclusions [clj-time/clj-time]} - ring-basic-authentication/ring-basic-authentication {:mvn/version "1.1.1"} - ring/ring-ssl {:mvn/version "0.3.0"} - bidi/bidi {:mvn/version "2.1.6" :exclusions [prismatic/schema]} - ;; Taking out storm cause yeah... - ;[org.apache.storm/storm-core "0.9.2-incubating"] - ;; Can't find bouncycastle trying to use this - ;incanter {:mvn/version "1.9.3" :exclusions [org.clojure/clojure]} bigml/sampling {:mvn/version "3.2"} - com.cognitect.aws/api {:mvn/version "0.8.641"} - com.cognitect.aws/endpoints {:mvn/version "1.1.12.398"} - com.cognitect.aws/s3 {:mvn/version "825.2.1250.0"} - org.postgresql/postgresql {:mvn/version "42.5.2"} + org.postgresql/postgresql {:mvn/version "42.7.4"} korma/korma {:mvn/version "0.4.3"} ;; Enhanced connection pooling - com.zaxxer/HikariCP {:mvn/version "5.0.1"} + com.zaxxer/HikariCP {:mvn/version "5.1.0"} clj-time/clj-time {:mvn/version "0.15.2"} - ;clj-excel {:mvn/version "0.0.1"} semantic-csv/semantic-csv {:mvn/version "0.2.0"} - ;[dk.ative/docjure {:mvn/version "1.13.0"] prismatic/plumbing {:mvn/version "0.6.0"} environ/environ {:mvn/version "1.2.0"} mount/mount {:mvn/version "0.1.17"} @@ -57,7 +39,7 @@ org.clojure/test.check {:mvn/version "1.1.1"} irresponsible/tentacles {:mvn/version "0.6.9"} techascent/tech.ml.dataset {:mvn/version "6.104"}} - :aliases {:dev + :aliases {:dev {:extra-paths ["dev" "test"] :extra-deps {cider/cider-nrepl {:mvn/version "0.30.0"} metasoarous/oz {:mvn/version "2.0.0-alpha5"}} @@ -71,7 +53,6 @@ :test {:extra-paths ["test"] :main-opts ["-m" "test-runner"]}} - ;:jvm-opts ^:replace [] :jvm-opts ["-Xmx4g"]} - ;:mvn/repos {"twitter4j" {:url "https://twitter4j.org/maven2"}}} - + + \ No newline at end of file diff --git a/math/dev.env.sh b/math/dev.env.sh index daae5c9ec2..062edde75c 100755 --- a/math/dev.env.sh +++ b/math/dev.env.sh @@ -2,4 +2,3 @@ export DATABASE_IGNORE_SSL=true export DATABASE_URL=postgres://postgres:oiPorg3Nrz0yqDLE@localhost:5432/polis-dev -export MATH_ENV=dev diff --git a/math/dev/user.clj b/math/dev/user.clj index 46081f5315..f64c2e5bf7 100644 --- a/math/dev/user.clj +++ b/math/dev/user.clj @@ -1,38 +1,22 @@ (ns user - (:require [polismath.runner :as runner] - [polismath.system :as system] - [polismath.meta.microscope :as microscope] - [polismath.conv-man :as conv-man] - [polismath.math.conversation :as conv] - [polismath.components.postgres :as postgres] - [polismath.math.named-matrix :as nm] - [polismath.math.corr :as corr] - [polismath.tasks :as tasks] - [plumbing.core :as plmb] - [honeysql.core :as honey] - [clojure.core.matrix :as matrix] - [clojure.core.async :as async :refer [>! !! runner/system :conversation-manager :conversations deref (get zid) :conv))) - (defn load-conv [{:as args :keys [zid zinvite]}] (let [zid (or zid (postgres/get-zid-from-zinvite (:postgres runner/system) zinvite))] @@ -44,10 +28,10 @@ (defn cmnt-group-votes [conv tid] (into {} - (map - (fn [[gid data]] - [(str "g-"gid "-votes") (str (get-in data [:votes tid]))]) - (:group-votes conv)))) + (map + (fn [[gid data]] + [(str "g-" gid "-votes") (str (get-in data [:votes tid]))]) + (:group-votes conv)))) (defn importance-metric @@ -63,109 +47,109 @@ {:as total-votes :keys [A D S P]} ;; reduce over votes per group, already aggregated (reduce - (fn [votes [gid data]] - (let [{:as data :keys [A S D]} (get-in data [:votes tid]) - data (assoc data :P (+ (- S (+ A D))))] - ;; Add in each of the data's kv count pairs - (reduce - (fn [votes' [k v]] - (update votes' k + v)) - votes - data))) - {:A 0 :D 0 :S 0 :P 0} - group-votes) + (fn [votes [gid data]] + (let [{:as data :keys [A S D]} (get-in data [:votes tid]) + data (assoc data :P (+ (- S (+ A D))))] + ;; Add in each of the data's kv count pairs + (reduce + (fn [votes' [k v]] + (update votes' k + v)) + votes + data))) + {:A 0 :D 0 :S 0 :P 0} + group-votes) importance (importance-metric A P S extremity) priority (get-in conv [:comment-priorities tid])] (log/spy - {:total-votes (str total-votes) - :priority priority - :importance importance - :fontSize (* priority 40)}))) + {:total-votes (str total-votes) + :priority priority + :importance importance + :fontSize (* priority 40)}))) (defn comments-data [conv] - (->> - (map - (fn [tid extremity [x y]] - (merge - {:tid tid - :label tid - :x x - :y y - :extremity extremity - :type "comment"} - (cmnt-group-votes conv tid) - (cmnt-stats conv tid extremity))) - (:tids conv) - (-> conv :pca :comment-extremity) - (-> conv :pca :comment-projection matrix/transpose)) - (remove (comp (:mod-out conv) :tid)))) + (->> + (map + (fn [tid extremity [x y]] + (merge + {:tid tid + :label tid + :x x + :y y + :extremity extremity + :type "comment"} + (cmnt-group-votes conv tid) + (cmnt-stats conv tid extremity))) + (:tids conv) + (-> conv :pca :comment-extremity) + (-> conv :pca :comment-projection matrix/transpose)) + (remove (comp (:mod-out conv) :tid)))) (defn groups-data [conv] (map - (fn [{:keys [id center members]}] - (let [[x y] (vec center) - pids (->> (:base-clusters conv) - (filter (comp (set members) :id)) - (mapcat :members))] - {:size (count pids) - :x x - :y y - :bids members - :pids pids - :type "group" - :label (str "g-" id)})) - (:group-clusters conv))) + (fn [{:keys [id center members]}] + (let [[x y] (vec center) + pids (->> (:base-clusters conv) + (filter (comp (set members) :id)) + (mapcat :members))] + {:size (count pids) + :x x + :y y + :bids members + :pids pids + :type "group" + :label (str "g-" id)})) + (:group-clusters conv))) (defn base-clusters-data [conv] (map - (fn [{:keys [id center members]}] - (let [[x y] (vec center)] - {:size (count members) - :x x - :y y - :pids members - :type "base-cluster" - :label ""})) - (:base-clusters conv))) + (fn [{:keys [id center members]}] + (let [[x y] (vec center)] + {:size (count members) + :x x + :y y + :pids members + :type "base-cluster" + :label ""})) + (:base-clusters conv))) (defn subgroup-clusters-data [conv] (mapcat - (fn [[gid subgroups]] - (map - (fn [{:keys [id center members parent-id]}] - (let [[x y] (vec center) - pids (->> (:base-clusters conv) - (filter (comp (set members) :id)) - (mapcat :members))] - {:size (count pids) - :x x - :y y - :bids members - :pids pids - :gid gid - :type "subgroup-cluster" - :parent parent-id - :label (str "g-" gid "-sg-" id)})) - subgroups)) - (:subgroup-clusters conv))) + (fn [[gid subgroups]] + (map + (fn [{:keys [id center members parent-id]}] + (let [[x y] (vec center) + pids (->> (:base-clusters conv) + (filter (comp (set members) :id)) + (mapcat :members))] + {:size (count pids) + :x x + :y y + :bids members + :pids pids + :gid gid + :type "subgroup-cluster" + :parent parent-id + :label (str "g-" gid "-sg-" id)})) + subgroups)) + (:subgroup-clusters conv))) (defn conv-data [conv] (concat - (comments-data conv) - (groups-data conv) - (subgroup-clusters-data conv) - (base-clusters-data conv))) + (comments-data conv) + (groups-data conv) + (subgroup-clusters-data conv) + (base-clusters-data conv))) (def size-scale @@ -175,8 +159,7 @@ (def conv-plot {:width 2000 :height 1300 - :layer [ - {:mark "rule" + :layer [{:mark "rule" :encoding {:x {:value 0 :scale {:zero false}}}} ;{:mark "rule" ; :encoding {:x {:value 0.0}}} @@ -201,8 +184,7 @@ (def conv-plot {:width 2000 :height 1300 - :layer [ - {:mark "rule" + :layer [{:mark "rule" :encoding {:x {:value 0 :scale {:zero false}}}} ;{:mark "rule" ; :encoding {:x {:value 0.0}}} @@ -230,41 +212,29 @@ ([conv] (oz/v! conv-plot :data {:values (conv-data conv)}))) -;(p! conv) - -;(oz/v! {:data {:values [{:x 1 :y 2} {:x 3 :y 4.3}]} - ;:layer [{:mark "point" - ;:encoding {:x {:field "x"} :y {:field "y"}}}]}) - -;; Test calls here -;(def plot-data -; (conv-data conv)) -;(def the-plot - ;(p! conv)) - (defn integrate [coll] (:result - (reduce - (fn [result x] - (-> result + (reduce + (fn [result x] + (-> result (update :total + x) (update :result conj (+ (:total result) x)))) - {:total 0 :result []} - coll))) + {:total 0 :result []} + coll))) (defn plot-priorities! [conv & {:keys [strict-mod exclude-meta]}] (let [values (->> (:comment-priorities conv) (remove (comp - (set/union - (if exclude-meta (:meta-tids conv) #{}) - (if strict-mod - (set/difference (set (:tids conv)) (:mod-in conv)) - (set/difference (:mod-out conv) (:meta-tids conv)))) - first)) + (set/union + (if exclude-meta (:meta-tids conv) #{}) + (if strict-mod + (set/difference (set (:tids conv)) (:mod-in conv)) + (set/difference (:mod-out conv) (:meta-tids conv)))) + first)) (sort-by second)) max-integral (apply max (integrate (vals values))) @@ -273,54 +243,28 @@ (comments-data conv))) entities (map - (fn [i [tid x] X] - (merge - (get comments tid) - {:rank i - :rank-perc (/ i (count values)) - :tid tid - :priority x - :integral X - :prob (/ X max-integral) - :is-meta (boolean (get (:meta-tids conv) tid)) - :mod-out (boolean (get (:mod-out conv) tid))})) - (range) - values - (integrate (vals values)))] + (fn [i [tid x] X] + (merge + (get comments tid) + {:rank i + :rank-perc (/ i (count values)) + :tid tid + :priority x + :integral X + :prob (/ X max-integral) + :is-meta (boolean (get (:meta-tids conv) tid)) + :mod-out (boolean (get (:mod-out conv) tid))})) + (range) + values + (integrate (vals values)))] (oz/v! - {:data {:values entities} - :title "Priority CDF" - :width 1400 - :height 900 - :mark "bar" - :encoding {:x {:field "rank-perc"} - :y {:field "prob"}}}))) - ;:y {:field "priority"}}}))) - - - -(defn prob-dist - "Messy function for investigating statistical distributions. Given a sample of comments selected - from the server's get comments fn, or from a page reload, this function returns the subset of those - comments which fall in the uppermost percentile of comments. By tracing back y values on a cdf plot to - the x values at which they intersect the cft, you can get a sense of what portion of sample should fall - within the specified percentile and then compare that to what you get here." - [conv sample percentile & {:keys [in-conv]}] - (let [comment-priorities - (->> (:comment-priorities conv) - (sort-by second) - (remove (comp (set/difference (:mod-out conv) (:meta-tids conv)) - first)) - (filter (comp - (set (or in-conv (:tids conv))) - first)))] - (log/info (count (vec comment-priorities))) - (set/intersection - (into #{} - (map first - (take-last (log/spy :info (int (* (count comment-priorities) percentile))) - comment-priorities))) - (set sample)))) + {:data {:values entities} + :title "Priority CDF" + :width 1400 + :height 900 + :mark "bar" + :encoding {:x {:field "rank-perc"} + :y {:field "prob"}}}))) (defn run-with-repl @@ -336,10 +280,10 @@ (comment (oz/start-plot-server!) ;; Run one of these to interactively test out a particular system or subsystem - (runner/run! system/base-system {:math-env :preprod}) - ;(runner/run! system/poller-system {:math-env :dev :poll-from-days-ago 0.1}) - ;(runner/run! system/task-system {:math-env :preprod :poll-from-days-ago 3}) - ;(runner/run! system/full-system {:math-env :preprod :poll-from-days-ago 0.1}) + (runner/run! system/base-system) + ;(runner/run! system/poller-system {:poll-from-days-ago 0.1}) + ;(runner/run! system/task-system {:poll-from-days-ago 3}) + ;(runner/run! system/full-system {:poll-from-days-ago 0.1}) ;(runner/run! system/darwin-system) @@ -371,17 +315,6 @@ (->> (nm/get-matrix (:raw-rating-mat conv)) matrix/shape) (sort (keys conv)) - ;(let [matrix (nm/get-matrix (:raw-rating-mat conv))] - ;(->> matrix - ;matrix/columns - ;(map - ;(fn [col] - ;(let [seen (remove nil? col) - ;agree ()] - ;#(/ % (-> matrix matrix/shape first)) - ;count - ;double))) - ;sort)) ;; Look at profile output (sort-by (comp - second) @(:profile-data conv)) @@ -390,9 +323,6 @@ ;; Plot the conversation (p! conv) - ;; Run correlation matrix - ;(corr/default-tids conv) - ;; Let's try another conversation (def zid2 17023) @@ -405,64 +335,42 @@ zid2 []) - ;; test queuing report task - ;(conv-man/queue-message-batch! (:conversation-manager runner/system) - ; :generate_report_data - ; zid2 - ; ;; these are probably wrong... - ; [{:rid 6 :zid 15228 :math_tick 52}]) - - - ;; Testing out updates via the pure conv-update function - (let [updated-conv (conv/conv-update conv [{:zid 15228 :pid 0 :tid 0 :vote 2.0 :created (System/currentTimeMillis)}]) - _ (printn "colnames") - updated-conv (conv/mod-update conv [{:zid 15230 :tid 38 :is_meta true :modified (System/currentTimeMillis)}]) - _ (log/info "First update") - updated-conv' (conv/conv-update updated-conv [{:zid 15228 :pid 0 :tid 0 :vote -1 :created (System/currentTimeMillis)}]) - _ (log/info "Second update")] - (log/info "Conv keys:" (keys conv)) - (log/info "Updated conv keys:" (keys updated-conv)) - (log/info "Previous conv key:" (boolean (:conv updated-conv))) - (log/info "Previous conv key:" (boolean (:conv (:conv updated-conv')))) - (:subgroup-clusters updated-conv')) - ;(:repness updated-conv')) - ;; Postgres/db testbench (postgres/query - (:postgres runner/system) - ["select * from votes + (:postgres runner/system) + ["select * from votes limit 10;"]) (postgres/query - (:postgres runner/system) - ["select * from votes + (:postgres runner/system) + ["select * from votes where zid = ?;" - 4]) + 4]) (postgres/query - (:postgres runner/system) - ;; Builds a string like the above - (honey/format - {:select [:*] - :from [:votes] - :limit 10})) + (:postgres runner/system) + ;; Builds a string like the above + (honey/format + {:select [:*] + :from [:votes] + :limit 10})) ;; This is the preferred way (postgres/query - (:postgres runner/system) - ;; Processes maps automatically, so you can do either of the two - {:select [:*] - :from [:votes] - :limit 10}) + (:postgres runner/system) + ;; Processes maps automatically, so you can do either of the two + {:select [:*] + :from [:votes] + :limit 10}) ;; Debugging issue (postgres/query - (:postgres runner/system) - {:select [:*] - :from [:math_tasks] - :limit 10}) + (:postgres runner/system) + {:select [:*] + :from [:math_tasks] + :limit 10}) ;; Getting config settings @@ -479,11 +387,4 @@ (doseq [i (range size)] (reduce + (range i)))) - (let [kill-chan (async/promise-chan) - parallelism 4 - work-size 30000] - (doseq [_ (range parallelism)] - (thread - (dowork work-size)))) - :end) diff --git a/math/doc/commands.md b/math/doc/commands.md index 23646c3a24..e6da3a9d11 100644 --- a/math/doc/commands.md +++ b/math/doc/commands.md @@ -30,7 +30,6 @@ The Polismath system provides several subcommands that run different components | `poller` | poller-system | Runs the polling system to watch for new votes and moderation | | `tasks` | task-system | Runs the task system for auxiliary jobs | | `full` | full-system | Runs both poller and task systems together | -| `simulator` | simulator-system | Runs a simulation system (for development/testing) | | `export` | export-system | Exports conversation data to files | These subcommands are defined in the `polismath.runner` namespace, with execution paths determined by the `-main` function. @@ -58,7 +57,7 @@ The `export` command has many additional options: -f, --filename FILENAME Name of output file (should be zip for csv out) -t, --at-time AT_TIME A string of YYYY-MM-DD-HH-MM-SS (in UTC) or ms-timestamp since epoch -T, --at-times AT_TIMES A vector of strings of --at-time format --F, --format FORMAT Either csv, excel or (soon) json +-F, --format FORMAT Either csv or json -M, --update-math Update math -P, --update-postgres Update postgres -h, --help Print help and exit @@ -79,8 +78,7 @@ When working with the REPL, you can run and manage systems directly using functi (runner/run! system/full-system) ;; Run with configuration overrides -(runner/run! system/full-system {:math-env :preprod - :poll-from-days-ago 0.1}) +(runner/run! system/full-system {:poll-from-days-ago 0.1}) ;; Run just the poller system (runner/run! system/poller-system) diff --git a/math/doc/configuration.md b/math/doc/configuration.md index 0140812356..0d8baf4d0e 100644 --- a/math/doc/configuration.md +++ b/math/doc/configuration.md @@ -62,7 +62,6 @@ Environment variables are the primary method for configuring the system. When us | Variable | Description | Default | |----------|-------------|---------| -| `MATH_ENV` | Environment setting (`dev`, `prod`, `preprod`) | `dev` | | `LOGGING_LEVEL` | Logging level (`trace`, `debug`, `info`, `warn`, `error`, `fatal`, `report`) | `warn` | | `POLL_FROM_DAYS_AGO` | How far back to poll for conversation updates (in days) | 10 | @@ -96,20 +95,6 @@ Environment variables are the primary method for configuring the system. When us | `EXPORT_SERVER_AUTH_USERNAME` | Username for export server authentication | None | | `EXPORT_SERVER_AUTH_PASS` | Password for export server authentication | None | -### AWS Configuration (for exports) - -| Variable | Description | Default | -|----------|-------------|---------| -| `AWS_ACCESS_KEY` | AWS access key ID | None | -| `AWS_SECRET_KEY` | AWS secret access key | None | - -### API Authentication - -| Variable | Description | Default | -|----------|-------------|---------| -| `WEBSERVER_USERNAME` | Username for webserver authentication | None | -| `WEBSERVER_PASS` | Password for webserver authentication | None | - ## Setting Environment Variables ### Docker Environment @@ -118,7 +103,6 @@ With Docker Compose, you can use a `.env` file: ```sh DATABASE_URL=postgres://username:password@hostname:port/database -MATH_ENV=dev MATH_LOG_LEVEL=info # Docker Compose translates this to LOGGING_LEVEL ``` @@ -134,7 +118,6 @@ When running without Docker, set environment variables directly: ```bash export DATABASE_URL=postgres://username:password@hostname:port/database -export MATH_ENV=dev export LOGGING_LEVEL=info # Use LOGGING_LEVEL directly when not using Docker Compose clojure -M:dev ``` @@ -149,8 +132,7 @@ When using the REPL, you can override configuration when starting the system: ;; Run with custom configuration (runner/run! system/full-system - {:math-env :preprod - :poll-from-days-ago 0.1}) + {:poll-from-days-ago 0.1}) ``` ## Logging Configuration @@ -197,7 +179,7 @@ To run a specific subsystem, use: ```clojure ;; In REPL -(runner/run! system/poller-system {:math-env :dev}) +(runner/run! system/poller-system) ;; Or via command line clojure -M:run poller diff --git a/math/doc/update.md b/math/doc/update.md index 1876d5663c..e913982fb7 100644 --- a/math/doc/update.md +++ b/math/doc/update.md @@ -8,7 +8,7 @@ Okay, I've reviewed the provided code, focusing on the `conv-update` functionali The conversation update process is primarily driven by these functions: -* **`user.clj`:** This seems to be a general utility and REPL interaction namespace. It includes functions for loading conversations (`load-conv`, `get-conv`), plotting, and some data analysis. It's the entry point for many manual operations. +* **`user.clj`:** This seems to be a general utility and REPL interaction namespace. It includes functions for loading conversations (`load-conv`), plotting, and some data analysis. It's the entry point for many manual operations. * **`polismath.runner/run!`:** The main entry point for starting the system. It initializes and starts components. * **`polismath.conv_man/queue-message-batch!`:** This function queues messages (votes, moderation actions, report generation requests) for processing by a conversation actor. It's the main way the outside world interacts with a conversation. * **`polismath.conv_man/conv-actor`:** Creates a core.async go-loop that processes messages for a _single_ conversation. It maintains the conversation state in an atom (`:conv`). @@ -133,7 +133,7 @@ The conversation update process is primarily driven by these functions: The use of `matrix/pow` here, even with a small exponent (2), might be more computationally expensive than simple multiplication. Consider replacing `(matrix/pow x 2)` with `(* x x)`. -* **`take-all!` and `take-all!!`:** These macros/functions are generally fine, but be aware of the potential for blocking if the channel is constantly being filled. +* **`take-all!`:** These macros/functions are generally fine, but be aware of the potential for blocking if the channel is constantly being filled. **Recommendations Summary** diff --git a/math/example.env b/math/example.env index 6fae261e01..b61b404a63 100644 --- a/math/example.env +++ b/math/example.env @@ -4,9 +4,8 @@ # cp example.env .env, modify as needed export DATABASE_IGNORE_SSL=true -export DATABASE_URL=DATABASE_URL=postgres://postgres:oiPorg3Nrz0yqDLE@localhost:5432/polis-dev +export DATABASE_URL=postgres://postgres:oiPorg3Nrz0yqDLE@localhost:5432/polis-dev export DD_ENABLED=false export LOGGING_LEVEL=debug -export MATH_ENV=dev export MATH_ZID_ALLOWLIST= export RECOMPUTE=true diff --git a/math/perf b/math/perf deleted file mode 100644 index f38ac1e1ff..0000000000 --- a/math/perf +++ /dev/null @@ -1,324 +0,0 @@ - -The following was computed on the a 5000x10 conversation; it seems that reflection is one of our problems here - - 2014-May-09 11:27:06 -0700 fullerine INFO [polismath.simulation] - Profiling: :user/clusters - Id Calls Min Max MAD Mean Time% Time - :user/clean-start-clusters 2 9.0ms 4.4s 2.2s 2.2s 30 4.4s - :user/safe-recenter-clusters 2 5.0ms 2.2s 1.1s 1.1s 15 2.2s - :user/row-subset 106 260.0μs 68.0ms 8.0ms 20.0ms 15 2.2s - :user/recenter-clusters 2 3.0ms 2.1s 1.1s 1.1s 14 2.1s - :user/rowname-subset 53 462.0μs 47.0ms 8.0ms 20.0ms 7 1.0s - :user/posible-clusters 2 306.0μs 23.0ms 12.0ms 12.0ms 0 24.0ms - :user/uniqify-clusters 2 20.0μs 1.0ms 737.0μs 757.0μs 0 2.0ms - Clock Time 100 14.8s - Accounted Time 81 12.0s - -Here are the reflection warnings that matter to our lib - - Auto-boxing loop arg: current-indent - exception.clj:44 recur arg for primitive local: i is not matching primitive, had: Object, needed: long - Auto-boxing loop arg: i - - # The main culprit? - Reflection warning, polismath/named_matrix.clj:21:41 - call to indexOf can't be resolved. - Reflection warning, polismath/named_matrix.clj:46:11 - call to indexOf can't be resolved. - Reflection warning, polismath/named_matrix.clj:55:11 - call to indexOf can't be resolved. - Reflection warning, polismath/named_matrix.clj:61:34 - call to indexOf can't be resolved. - Reflection warning, polismath/named_matrix.clj:67:24 - call to indexOf can't be resolved. - - pca.clj:51 recur arg for primitive local: last_eigval is not matching primitive, had: Object, needed: long - Auto-boxing loop arg: last-eigval - - Reflection warning, polismath/clusters.clj:136:27 - call to indexOf can't be resolved. - Reflection warning, polismath/clusters.clj:136:27 - call to indexOf can't be resolved. - Performance warning, polismath/clusters.clj:201:25 - case has int tests, but tested expression is not primitive. - Reflection warning, polismath/conversation.clj:22:13 - call to indexOf can't be resolved. - Reflection warning, polismath/simulation.clj:78:3 - reference to field nextElement can't be resolved. - - -Fuck... this doesn't seem to be working. I took away all of the reflection warnings with a type-indexof -function, but still no dice. What's left of the reflections: - - (defn typed-indexof [^java.util.List coll item] - (.indexOf coll item)) - - # Other libraries... - Reflection warning, /tmp/form-init1039726133741870246.clj:1:911 - call to invokeStaticMethod can't be resolved. - Reflection warning, alex_and_georges/debug_repl.clj:101:8 - reference to field getCause can't be resolved. - Reflection warning, alex_and_georges/debug_repl.clj:102:8 - reference to field getCause can't be resolved. - Reflection warning, plumbing/fnk/schema.clj:93:37 - call to java.lang.IllegalArgumentException ctor can't be resolved. - Reflection warning, clojure/tools/trace.clj:231:9 - call to setStackTrace can't be resolved. - Reflection warning, clojure/tools/trace.clj:242:11 - call to setStackTrace can't be resolved. - Reflection warning, clojure/tools/trace.clj:244:49 - call to java.lang.Exception ctor can't be resolved. - Reflection warning, clojure/tools/trace.clj:244:11 - call to setStackTrace can't be resolved. - Reflection warning, clojure/tools/trace.clj:276:9 - call to setStackTrace can't be resolved. - Reflection warning, clojure/tools/trace.clj:277:15 - call to setStackTrace can't be resolved. - Reflection warning, bigml/sampling/random.clj:11:11 - call to cern.jet.random.tdouble.engine.MersenneTwister64 vector can't be resolved. - Reflection warning, clj_time/core.clj:577:10 - reference to field getDayOfMonth can't be resolved. - - # Us???? - columns.clj:125 recur arg for primitive local: current_indent is not matching primitive, had: Object, needed: long - Auto-boxing loop arg: current-indent - exception.clj:44 recur arg for primitive local: i is not matching primitive, had: Object, needed: long - Auto-boxing loop arg: i - pca.clj:51 recur arg for primitive local: last_eigval is not matching primitive, had: Object, needed: long - Auto-boxing loop arg: last-eigval - - Performance warning, polismath/clusters.clj:201:25 - case has int tests, but tested expression is not primitive. - Reflection warning, polismath/conversation.clj:22:13 - call to indexOf can't be resolved. - Reflection warning, polismath/simulation.clj:78:3 - reference to field nextElement can't be resolved. - - -May 12, 2014 - -WOOT! It seems I've sorted out the biggest culrpit here. -It turned out to be the filter-by-index function. -It was creating a new set each time, which was absolutely killing performance. -I was able to cut down an order of magnitude by doing this alone. - -So now on to other things... - -For starters, I tried running on 100,000 ptpts 10 cmts. -There are the big and big-part update times - - N-ptpts: 99998 - Starting new conv update! - 1399963793428 rating-mat 1292120.671242 msecs #=> 1,292.120s #=> 20 minutes - 1399963793436 mat 7.876006 msecs - 1399963800685 pca 7249.400704 msecs # 7 sec - 1399963802045 proj 1359.328171 msecs # 1.3 sec - 1399963848873 base-clusters 46828.255953 msecs # 46 sec - 1399963848873 bid-to-pid 0.054336 msecs - 1399963849626 votes-base 752.377173 msecs - 1399963849626 counting-ptpts 0.003771 msecs - 1399963849628 group-clusters 2.392552 msecs - 1399963849628 CONVUP big 1353257.322353 msecs - - N-ptpts: 99998 - 1399963849843 rating-mat 140.33363 msecs - 1399963849849 mat 5.574799 msecs - 1399963855133 pca 5283.952589 msecs - 1399963856372 proj 1238.580412 msecs - 1399964442254 base-clusters 585882.249774 msecs # 585.882 # 10 min - 1399964442254 bid-to-pid 0.056502 msecs - 1399964442916 votes-base 661.268519 msecs - 1399964442916 counting-ptpts 0.003702 msecs - 1399964442917 group-clusters 1.644483 msecs - 1399964442918 CONVUP big-part 593284.9938 msecs - - 2014-May-13 00:00:42 -0700 noether INFO [polismath.simulation] - Profiling: :user/clusters - Id Calls Min Max MAD Mean Time% Time - :user/kmeans 2 2.0ms 585.9s 292.9s 292.9s 99 585.9s - :user/clean-start-clusters 2 1.0ms 546.2s 273.1s 273.1s 92 546.2s - :user/safe-recenter-clusters 2 604.0μs 276.2s 138.1s 138.1s 47 276.2s - :user/safe-recenter-map 2 552.0μs 276.2s 138.1s 138.1s 47 276.2s - :user/safe-rowname-subset 53 90.0μs 9.6s 1.7s 5.2s 46 274.5s - :user/recenter-clusters 2 485.0μs 269.9s 135.0s 135.0s 45 269.9s - :user/rowname-subset 53 82.0μs 8.7s 1.6s 5.1s 45 268.4s - :user/row-subset 106 46.0μs 47.0ms 3.0ms 28.0ms 1 3.0s - :user/filter-by-index 212 5.0μs 862.0μs 129.0μs 439.0μs 0 93.0ms - :user/uniqify-clusters 2 12.0μs 1.0ms 510.0μs 522.0μs 0 1.0ms - :user/safe-recenter-empty 2 3.0μs 301.0μs 149.0μs 152.0μs 0 303.0μs - :user/safe-recenter-nil 2 21.0μs 155.0μs 67.0μs 88.0μs 0 175.0μs - Clock Time 100 593.3s - Accounted Time 421 2500.3s - - -Hmm.... This is suggesting that performance on the rating-mat update may actually become a problem if we can't -keep everything else moving quickly enough. -Of course, this corresponds to 1,000,000 votes. -If we ever get 1,000,000 votes at once then (for a specific conv), God help us for now... XXX - -On the other hand, note that the base clustering takes QUITE a long time in this second one here (10 minutes). -Again, this seems to be due largely due to the clean-start-clusters. -We really have to figure out how to speed that up. XXX - -Another thought: initially I was concerned that something might not be quite right given that the second -(partial) updata would always take so much longer than the full matrix population. -But this actually makes sense: most of the time is in the clean-start-clusters, but the clean-start-clusters -is going to be a lot simpler if the last round didn't have that many members. - -Also, note that there are still some strange "unable to slice nil" errors cropping up every now and again. -A few of those leading to a backlog, and it might put us on ice while the conversation catches up. XXX - -One last thing: -Note that the PCA grows as a function of the number of comments, but that downstream analyses don't, because -they're just on projections. - - # 10 cmts - big - 1399967451528 pca 942.879957 msecs # 1 sec - 1399967451706 proj 177.306975 msecs # 0.2 sec - big-part - 1399967458208 pca 751.15802 msecs # < 1sec - 1399967458369 proj 161.097721 msecs - - # 100 cmts - big - 1399967669033 pca 6840.381594 msecs # 7 sec - 1399967669866 proj 833.318748 msecs # 0.8 sec - big-part - 1399967679124 pca 3850.089523 msecs # 4 sec - 1399967679911 proj 786.742437 msecs # 0.8 sec - -Something interesting here: Note the second times are a lot shorter in each case (1/2-ish). -This is likely because the power-iteration is working exactly as planned. -Which is good. -Still, if it's another 7x to increase both the cmts and ptpts 10x, then we could easily expect to see minute long -compute times for our PCA step. -So we'll either want to fix our Batch PCA or try to speed this bit up XXX - -Also note here that the time for update is much lower than it was when there were 10x more people (80sec). -Could this let us use a tranpose for this messy operation? XXX -In general this may also point to where some of the possible optimizations are with respect to this update -implementation. - - -May 18, 2014 - -Been a little while since I checked in... - -I've figured out what is causing the slow rating mtarix update. -Silly me... I was doing indexOf on the rows list to get to the matrix indices. -Bad idea... -I'm going to switch to hash-maos, through which I think we can cut the matrix-update code to seconds with 500k ptpts -This was what we had from that count: - - - 2014-May-17 22:27:26 -0700 noether INFO [polismath.simulation] - Profiling: :user/clusters - Id Calls Min Max MAD Mean Time% Time - :user/update-nmat 1 392.2s 392.2s 0ns 392.2s 92 392.2s - :user/MU-indices 600000 2.0μs 3.0ms 324.0μs 627.0μs 88 376.2s - :user/rowname-subset 54 47.0μs 15.7s 571.0ms 298.0ms 4 16.1s - :user/kmeans 2 2.0ms 12.1s 6.1s 6.1s 3 12.1s - :user/MU-rows-cols 600000 1.0μs 1.3s 4.0μs 4.0μs 1 2.6s - :user/clean-start-clusters 2 1.0ms 1.6s 818.0ms 819.0ms 0 1.6s - :user/MU-assoc-in 600000 1.0μs 4.0ms 278ns 2.0μs 0 1.4s - :user/MU-indices-2 600000 1.0μs 219.0μs 247ns 2.0μs 0 1.2s - :user/safe-recenter-clusters 2 576.0μs 820.0ms 410.0ms 410.0ms 0 820.0ms - :user/safe-recenter-map 2 564.0μs 819.0ms 409.0ms 410.0ms 0 820.0ms - :user/row-subset 107 29.0μs 45.0ms 988.0μs 7.0ms 0 771.0ms - :user/recenter-clusters 2 351.0μs 768.0ms 384.0ms 384.0ms 0 768.0ms - :user/MU-assoc 600000 698ns 1.0ms 226ns 1.0μs 0 626.0ms - :user/MU-row-add 600000 488ns 197.0μs 492ns 966ns 0 580.0ms - :user/MU-col-add 600000 488ns 293.0μs 101ns 911ns 0 547.0ms - :user/safe-rowname-subset 53 57.0μs 12.0ms 856.0μs 8.0ms 0 401.0ms - Clock Time 100 428.3s - Accounted Time 189 808.8s - -Will want to keep an eye on the other MU items on that list, as they are liable to be the next points for speeding things up. - - -May 21, 2014 - -Woot! I seem to have stamped this shit. -At least largely. -Before I go on though, I should make a correction. -I'm pretty sure the above was actually on 600,000 new _comments_, not ptpts. -The latter would not make sense with the call counts shown above. - -Anyway; it seems that using indexOf on matrices was basically just a terrible idea. -I thought lookup was constant time. -Which is silly in retrospect. -Anyway. -It's not. - -I've implemented a custom data type which maintains both a vector and a hash-map so we can quickly get (the -former equivalent of) indexOf. -Now look at the times: - - Starting new conv update! - N-ptpts: 60000 - count 720000 - 1400735231267 rating-mat 3003.265079 msecs - 1400735231267 counting-comments 0.0044 msecs - 1400735231336 user-vote-count 68.865603 msecs - 1400735231381 mat 5.540164 msecs - 1400735235871 pca 4489.99813 msecs - 1400735236737 proj 865.464804 msecs - 1400735256213 base-clusters 19475.856953 msecs - 1400735256213 bid-to-pid 0.054686 msecs - 1400735256715 votes-base 501.544857 msecs - 1400735256715 counting-ptpts 0.004539 msecs - 1400735256717 group-clusters 1.903313 msecs - 1400735256717 CONVUP big-part 31481.174735 msecs - 2014-May-21 22:07:36 -0700 noether INFO [polismath.simulation] - Profiling: :user/clusters - Id Calls Min Max MAD Mean Time% Time - :user/kmeans 2 2.0ms 19.3s 9.6s 9.6s 61 19.3s - :user/update-nmat 1 2.9s 2.9s 0ns 2.9s 9 2.9s - :user/clean-start-clusters 2 928.0μs 1.8s 921.0ms 921.0ms 6 1.8s - :user/safe-recenter-clusters 2 505.0μs 944.0ms 472.0ms 472.0ms 3 945.0ms - :user/safe-recenter-map 2 492.0μs 944.0ms 472.0ms 472.0ms 3 944.0ms - :user/recenter-clusters 2 268.0μs 837.0ms 418.0ms 418.0ms 3 837.0ms - :user/safe-rowname-subset 53 37.0μs 4.0ms 345.0μs 3.0ms 1 164.0ms - Clock Time 100 31.5s - Accounted Time 85 26.9s - - -Damn son. -Down to 30 seconds. -Still some room for improvment given I'm looking at some new reflection/performance warnings. -We'll see how things do after that. - - -Hmmm... I tried running on 1mil ptpts and 100 cmts and got this. - - Starting new conv update! - N-ptpts: 100000 - count 10200000 - 1400736772092 rating-mat 454258.957733 msecs - 1400736772093 counting-comments 0.005029 msecs - 1400736778787 user-vote-count 6694.438309 msecs - 1400736778866 mat 8.482992 msecs - 1400736853808 pca 74942.795939 msecs - 1400736863400 proj 9591.412637 msecs - 1400736914508 base-clusters 51108.178211 msecs - 1400736914509 bid-to-pid 0.065022 msecs - Exception in thread "main" java.lang.OutOfMemoryError: GC overhead limit exceeded - at clojure.lang.PersistentHashMap$ArrayNode.assoc(PersistentHashMap.java:364) - at clojure.lang.PersistentHashMap$ArrayNode.assoc(PersistentHashMap.java:361) - at clojure.lang.PersistentHashMap.assoc(PersistentHashMap.java:142) - at clojure.lang.PersistentHashMap.assoc(PersistentHashMap.java:28) - at clojure.lang.RT.assoc(RT.java:702) - at clojure.core$assoc.invoke(core.clj:187) - at clojure.core$zipmap.invoke(core.clj:2715) - at polismath.conversation$agg_bucket_votes_for_tid.invoke(conversation.clj:24) - at polismath.conversation$fn__4283$fnk4280_positional__4284$fn__4287.invoke(conversation.clj:168) - - -Can probably fix this with JVM settings, but for now we seem to have found an upper limit. -However, of interest is that the rating-mat update, while significant is operating on 100mil votes, which -we'll liekly never see in a single update. -If we have a milion people probably more like 10 per user minute, tops. -So if we keep everything else moving quickly enough, we might be fine. - -Here is some data for 30k x 20cmts: - - Starting new conv update! - N-ptpts: 30000 - count 660000 - 1400746217528 rating-mat 1503.576119 msecs - 1400746217529 counting-comments 0.0044 msecs - 1400746217583 user-vote-count 54.830368 msecs - 1400746217621 mat 4.400001 msecs - 1400746221937 pca 4316.419774 msecs - 1400746222599 proj 661.168579 msecs - 1400746234367 base-clusters 11768.231347 msecs - 1400746234367 bid-to-pid 0.054615 msecs - 1400746234788 votes-base 420.764263 msecs - 1400746234788 counting-ptpts 0.004191 msecs - 1400746234790 group-clusters 1.557323 msecs - 1400746234790 CONVUP big-part 21241.605796 msecs - 2014-May-22 01:10:34 -0700 noether INFO [polismath.simulation] - Profiling: :user/clusters - Id Calls Min Max MAD Mean Time% Time - :user/kmeans 2 1.0ms 11.7s 5.8s 5.8s 55 11.7s - :user/update-nmat 1 1.4s 1.4s 0ns 1.4s 6 1.4s - :user/clean-start-clusters 2 891.0μs 1.2s 588.0ms 589.0ms 6 1.2s - :user/safe-recenter-clusters 2 470.0μs 585.0ms 292.0ms 293.0ms 3 585.0ms - :user/safe-recenter-map 2 457.0μs 585.0ms 292.0ms 293.0ms 3 585.0ms - :user/recenter-clusters 2 260.0μs 520.0ms 260.0ms 260.0ms 2 521.0ms - :user/safe-rowname-subset 53 33.0μs 3.0ms 222.0μs 2.0ms 0 105.0ms - Clock Time 100 21.2s - Accounted Time 75 16.0s - - diff --git a/math/python_conversion/.gitignore b/math/python_conversion/.gitignore deleted file mode 100644 index fed8a5d32c..0000000000 --- a/math/python_conversion/.gitignore +++ /dev/null @@ -1,54 +0,0 @@ -# Python bytecode -__pycache__/ -*.py[cod] -*$py.class - -# Distribution / packaging -dist/ -build/ -*.egg-info/ -*.egg - -# Virtual environments -polis_env/ -new_polis_env/ -venv/ -ENV/ -env/ -.env -.venv - -# Jupyter Notebook -.ipynb_checkpoints -*/.ipynb_checkpoints/* - -# Data files -data/ -*.csv -*.json -*.npy -*.pkl -*.db -*.sqlite - -# Development files -.idea/ -.vscode/ -*.swp -*.swo -.DS_Store - -# Pytest cache -.pytest_cache/ -.coverage -htmlcov/ - -# Logs -*.log -logs/ - -# Environment variables -.env - -# Generated files -*.so \ No newline at end of file diff --git a/math/resources/config.edn b/math/resources/config.edn index dedffca616..c5f785dd59 100644 --- a/math/resources/config.edn +++ b/math/resources/config.edn @@ -1,44 +1,22 @@ { ;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . - ;; TODO This is a work in progress... really think we need to move to aero though: https://github.com/juxt/aero - ;; Soon... - - :math-env #or [#keyword #env MATH_ENV, :dev] - :darwin {:server-port #or [#long #env SERVER_PORT, 3123]} - :database {:pool-size #or [#long #env DATABASE_POOL_SIZE, 3] :url #env DATABASE_URL} :math-schema-date "2014_08_22" :export {:expiry-days 10} - ;; TODO This poller config hasn't been hooked up yet... ghost :poller {:votes {:polling-interval 2000} :moderation {:polling-interval 5000}} :math {:matrix-implementation :vectorz :cutoffs {:medium 100 :large 10000}} - ;; These are copied over from the conv update opts; Should expose them through more generally - ;; A little messy because these get used in different places than above... - ;:n-comps 2 - ;:pca-iters 10 - ;:base-iters 10 - ;:base-k 50 - ;:max-k 5 - ;:group-iters 10 - ;:max-ptpts 80000 - ;:max-cmts 800 - ;:group-k-buffer 4}} :logging {:file "log/dev.log" :level :warn} - ;:math-env {:parse ->keyword} - ;; Have to use :port since that's what heroku expects... - ;:port {:path [:darwin :server-port] :parse ->long} - ;:database-url {:path [:database :url]} :database-for-reads-name {:path [:database :reads-name]} ;:database-pool-size {:path [:database :pool-size] :parse ->long} :mongolab-uri {:path [:mongo :url]} diff --git a/math/src/data_readers.clj b/math/src/data_readers.clj deleted file mode 100644 index 6e88088cd6..0000000000 --- a/math/src/data_readers.clj +++ /dev/null @@ -1,7 +0,0 @@ -;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . - -{mikera.vectorz.Vector clojure.core.matrix/matrix - mikera.arrayz.NDArray clojure.core.matrix/matrix - mikera.matrixx.Matrix clojure.core.matrix/matrix - ;clojure.core.matrix.impl.ndarray.NDArray clojure.core.matrix ;; Need to check this one... XXX - polismath.named-matrix.NamedMatrix polismath.named-matrix/named-matrix-reader} diff --git a/math/src/polismath/components/config.clj b/math/src/polismath/components/config.clj index b2a4eea571..54830448e3 100644 --- a/math/src/polismath/components/config.clj +++ b/math/src/polismath/components/config.clj @@ -1,15 +1,11 @@ ;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . (ns polismath.components.config - (:require [polismath.utils :as utils] - [taoensso.timbre :as log] - [com.stuartsierra.component :as component] - [environ.core :as environ] - [clojure.string :as string])) - - -;; I think we need to just move to this: https://github.com/juxt/aero -;; Good set of features without baking in too many assumptions or constraining design + (:require + [clojure.string :as string] + [com.stuartsierra.component :as component] + [environ.core :as environ] + [taoensso.timbre :as log])) (defn ->long [x] @@ -32,18 +28,12 @@ (not (#{"false" "0" "no"} x)))) -;; XXX This should be computed; that is be a function of the rules. -;; :default specification should be in the rules themselves (def defaults - {:math-env :dev - :math-schema-date "2014_08_22" + {:math-schema-date "2014_08_22" :server {:port 8080} - ;:darwin {:server-port 3123} :export {:expiry-days 6 :temp-dir "/tmp/" - ;; Hmmm should be able to specify a dep on port; aero? :private-url-base "http://localhost:8080"} - ;; Shit... should be able to specify dependency on preprod as well! :database {:pool-size 3} :poller {:votes {:polling-interval 1000} :moderation {:polling-interval 1000} @@ -62,25 +52,16 @@ (def rules "Mapping of env keys to parsing options" - {:math-env {:parse ->keyword} - ;; Have to use :port since that's what heroku expects... - :port {:path [:server :port] :parse ->long} + {:port {:path [:server :port] :parse ->long} :database-url {:path [:database :url]} :database-for-reads-name {:path [:database :reads-name]} :database-pool-size {:path [:database :pool-size] :parse ->long} :database-ignore-ssl {:path [:database :ignore-ssl] :parse ->boolean} - :mailgun-api-key {:path [:email :api-key]} - :mailgun-url {:path [:email :url]} - :aws-secret-key {:path [:aws :secret-access-key]} - :aws-access-key {:path [:aws :access-key-id]} - :webserver-username {:path [:webserver-username]} - :webserver-pass {:path [:webserver-pass]} :math-zid-blocklist {:path [:poller :zid-blocklist] :parse ->long-list} :math-zid-allowlist {:path [:poller :zid-allowlist] :parse ->long-list} :export-server-auth-username {:path [:darwin :server-auth-username]} :export-server-auth-pass {:path [:darwin :server-auth-pass]} :math-matrix-implementation {:path [:math :matrix-implementation] :parse ->keyword} - ;; TODO Put all these within a :conv-update opt so we can just pass that through to conv-update all at once :math-cutoff-medium {:path [:math :cutoffs :medium] :parse ->long :doc "This is the maximum size of a conversation before running in :medium mode"} :math-cutoff-large {:path [:math :cutoffs :large] :parse ->long @@ -90,53 +71,30 @@ :math-cutoff-max-cmnts {:path [:math :cutoffs :max-ptpts] :parse ->long :doc "This is the maximum number of comments before the conversation stops accepting new comments"} :math-schema-date {:doc "This helps us version our mongo buckets."} - ;; Should change these to be more abstract in key name; not hostedgraphite-apikey; just graphite-apikey etc XXX - :hostedgraphite-apikey {:path [:meta :graphite :api-key] - :doc "API key for graphite db (perf monitoring)"} - :hostedgraphite-hostname {:path [:meta :graphite :hostname] - :doc "The hostname for sending messages to graphite"} :export-expiry-days {:path [:export :expiry-days] :parse ->long - :doc "The number of days before a mongo record representing a data exports gets removed"} + :doc "The number of days before a data export record gets removed"} :vote-polling-interval {:parse ->long :path [:poller :votes :polling-interval] :doc "The polling interval for votes, in milliseconds"} :mod-polling-interval {:parse ->long :path [:poller :moderation :polling-interval] :doc "The polling interval for moderation, in milliseconds"} - ;:initial-polling-timestamp {:parse ->long :path [:poller :initial-polling-timestamp] - ; :doc "The initial vote and mod polling timestamp (only load convs with votes later than this)"} :poll-from-days-ago {:parse ->long :path [:poller :poll-from-days-ago]} - ;; Need to think more about the semantics of a recompute; once; always; only if not booted; etc? XXX :recompute {:parse ->boolean :doc "Whether or not to perform a recompute"} - ;; Need to think about how to handle options :logging-level {:path [:logging :level] :parse ->keyword :doc "Logging level for timbre; info, debug, error, etc"} :logging-file {:path [:logging :file] :doc "If set, a file to which the log will be appended"}}) - ;; XXX TODO & Thoughts - ;; Mini batch sizes (see polismath.math.conversation) - - -(defn assoc-inferred-values - [{:as config-map :keys [math-env]}] - (let [math-env-string (name math-env) - webserver-url (str "https://" - (when-not (= math-env :prod) (str math-env-string ".")) - "pol.is/api/v3")] - (assoc config-map - :math-env-string math-env-string - :webserver-url webserver-url))) - (defn get-environ-config [rules env] ;; reduce over rules and assoc-in mappings into empty map (reduce - (fn [config [name {:keys [parse path] :or {parse identity}}]] - (if-let [env-var-val (get env name)] - (assoc-in config (or path [name]) (parse env-var-val)) - config)) - {} - rules)) + (fn [config [name {:keys [parse path] :or {parse identity}}]] + (if-let [env-var-val (get env name)] + (assoc-in config (or path [name]) (parse env-var-val)) + config)) + {} + rules)) (defn deep-merge "Like merge, but merges maps recursively." @@ -147,12 +105,9 @@ (defn get-config ([overrides] - ;; Then infer additional values - (assoc-inferred-values - (deep-merge defaults - ;(read-string (slurp "config.edn")) - (get-environ-config rules environ/env) - overrides))) + (deep-merge defaults + (get-environ-config rules environ/env) + overrides)) ([] (get-config {}))) (defrecord Config [overrides] @@ -172,27 +127,3 @@ :ok - - -;; XXX This bit of refactoring is a WIP for switching to aero and mount (v component)... - -;(defmethod aero/reader `keyword -; [_ _ value] -; (keyword value)) -; -;(defn get-config -; ([profile overrides] -; (deep-merge (get-environ-conifg rules))) -; ([profile] (get-config profile {})) -; ([] (get-config :dev))) -; -;;; We use the overrides atom to allow the system start function to specify specific overrides -;(defonce profile (atom :dev)) -;(defonce overrides (atom nil)) -; -;(defstate config -; :start (do (log/info ">> Starting config component") -; (get-config profile overrides)) -; :stop (do (log/info "<< Stopping config component") -; (reset! overrides nil))) - diff --git a/math/src/polismath/components/core_matrix_boot.clj b/math/src/polismath/components/core_matrix_boot.clj index 6556f6bce5..3e3c126732 100644 --- a/math/src/polismath/components/core_matrix_boot.clj +++ b/math/src/polismath/components/core_matrix_boot.clj @@ -1,17 +1,13 @@ ;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . (ns polismath.components.core-matrix-boot - (:require [polismath.utils :as utils] - [clojure.core.matrix :as matrix] - [clojure.core.matrix.impl.ndarray :as ndarray] - [taoensso.timbre :as log] - [com.stuartsierra.component :as component] - [cheshire.generate :refer [add-encoder encode-seq remove-encoder]])) + (:require + [cheshire.generate :refer [add-encoder encode-seq]] + [clojure.core.matrix :as matrix] + [com.stuartsierra.component :as component] + [taoensso.timbre :as log])) -;; Maybe this just shouldn't be a component, but an environment thing? Or does it work cause things won't be loaded on -;; static compile anyway? Ug... - (defn matrix-encoder [v jsonGenerator] (encode-seq (into-array v) jsonGenerator)) @@ -62,8 +58,5 @@ ([options] (map->CoreMatrixBooter options))) - -;(component/start (create-core-matrix-booter {:config {:math {:matrix-implementation :vectorz}}})) - :ok diff --git a/math/src/polismath/components/env.clj b/math/src/polismath/components/env.clj deleted file mode 100644 index d7c6fb2fbe..0000000000 --- a/math/src/polismath/components/env.clj +++ /dev/null @@ -1,14 +0,0 @@ -;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . - -(ns polismath.components.env - (:require [environ.core :as environ])) - -;; Deprecating... Will remove by the end of refactor XXX - -(def ^:dynamic env environ/env) - -(defmacro with-env-overrides - [overrides & body] - `(binding [env (merge env ~overrides)] - ~@body)) - diff --git a/math/src/polismath/components/logger.clj b/math/src/polismath/components/logger.clj index f0517cfead..0118890473 100644 --- a/math/src/polismath/components/logger.clj +++ b/math/src/polismath/components/logger.clj @@ -1,9 +1,10 @@ ;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . (ns polismath.components.logger - (:require [com.stuartsierra.component :as component] - [taoensso.timbre :as log] - [taoensso.timbre.appenders.core :as appenders])) + (:require + [com.stuartsierra.component :as component] + [taoensso.timbre :as log] + [taoensso.timbre.appenders.core :as appenders])) ;; XXX Hmmm.. this doesn't seem to be working yet. Maybe we need to use timbre for log calls instead of @@ -16,22 +17,22 @@ (log/set-level! level)) ;; First merge the logging config from system config (log/merge-config! - (:logging config)) + (:logging config)) ;; Then merge appender config, inheriting the min-level from system config (log/merge-config! - {:appenders {:println-appender - {:enabled? true - :async? false - :min-level (get-in config [:logging :level] :warn) ; Use system config level or default to :warn - :rate-limit [[1 250] [10 5000]] ; 1/250ms, 10/5s - :output-fn :inherit - :fn ; Appender's fn - (fn [data] - (let [{:keys [output-fn]} data - formatted-output-str (output-fn data)] - (println formatted-output-str)))} - :file-appender - {:spit (appenders/spit-appender {:fname (get-in config [:logging :file] "dev.log")})}}}) + {:appenders {:println-appender + {:enabled? true + :async? false + :min-level (get-in config [:logging :level] :warn) ; Use system config level or default to :warn + :rate-limit [[1 250] [10 5000]] ; 1/250ms, 10/5s + :output-fn :inherit + :fn ; Appender's fn + (fn [data] + (let [{:keys [output-fn]} data + formatted-output-str (output-fn data)] + (println formatted-output-str)))} + :file-appender + {:spit (appenders/spit-appender {:fname (get-in config [:logging :file] "dev.log")})}}}) component) (stop [component] (log/info "<< Stopping config component") diff --git a/math/src/polismath/components/postgres.clj b/math/src/polismath/components/postgres.clj index a8b65f9668..1fe8a81f63 100644 --- a/math/src/polismath/components/postgres.clj +++ b/math/src/polismath/components/postgres.clj @@ -1,15 +1,16 @@ ;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . (ns polismath.components.postgres - (:require [cheshire.core :as cheshire] - [taoensso.timbre :as log] - [com.stuartsierra.component :as component] - [clojure.java.jdbc :as jdbc] - [honeysql.core :as sql] - [honeysql.helpers :as honey]) - (:import (org.postgresql.util PGobject) - (com.zaxxer.hikari HikariConfig HikariDataSource))) - ;[alex-and-georges.debug-repl :as dbr] + (:require + [cheshire.core :as cheshire] + [clojure.java.jdbc :as jdbc] + [com.stuartsierra.component :as component] + [honeysql.core :as sql] + [honeysql.helpers :as honey] + [taoensso.timbre :as log]) + (:import + (com.zaxxer.hikari HikariConfig HikariDataSource) + (org.postgresql.util PGobject))) @@ -107,7 +108,7 @@ pool-config (-> config :database)] (assert database-url "Missing database url. Make sure to set env variables.") (log/info "Configuring PostgreSQL connection pool with size:" (get pool-config :pool-size 10)) - (assoc component :db-spec (heroku-db-spec database-url + (assoc component :db-spec (heroku-db-spec database-url (-> config :database :ignore-ssl) pool-config)))) (stop [component] @@ -165,34 +166,34 @@ [component zinvite] (log/debug "get-zid-from-zinvite for zinvite" zinvite) (-> - (query component - {:select [:zid :zinvite] - :from [:zinvites] - :where [:= :zinvite zinvite]}) - first - :zid)) + (query component + {:select [:zid :zinvite] + :from [:zinvites] + :where [:= :zinvite zinvite]}) + first + :zid)) (defn get-meta-tids [component zid] (->> - (query component - {:select [:tid] - :from [:comments] - :where [:and [:= :zid zid] - :is_meta]}) - (map :tid) - (into #{}))) + (query component + {:select [:tid] + :from [:comments] + :where [:and [:= :zid zid] + :is_meta]}) + (map :tid) + (into #{}))) (defn get-zinvite-from-zid [component zid] (log/debug "get-zinvite-from-zid for zid" zid) (-> - (query component - {:select [:zid :zinvite] - :from [:zinvites] - :where [:= :zid zid]}) - first - :zinvite)) + (query component + {:select [:zid :zinvite] + :from [:zinvites] + :where [:= :zid zid]}) + first + :zinvite)) (defn conv-poll "Query for all vote data since last-vote-timestamp for a given zid, given an implicit db-spec" @@ -216,13 +217,13 @@ [component zid last-mod-timestamp] (log/info "conv-mod-poll for zid" zid ", last-vote-timestap" last-mod-timestamp) (query - component - {:select [:*] - :from [:comments] - :order-by [:tid :modified] - :where [:and - [:> :modified last-mod-timestamp] - [:= :zid zid]]})) + component + {:select [:*] + :from [:comments] + :order-by [:tid :modified] + :where [:and + [:> :modified last-mod-timestamp] + [:= :zid zid]]})) (defn format-as-json-for-db @@ -234,33 +235,27 @@ cheshire/generate-string cheshire/parse-string)) -; (defn collection-name -; "math_env name based on math-env and math-schema-date config variables. Makes sure that -; prod, preprod, dev (and subdevs like chrisdev or mikedev) have their own noninterfering collections." -; ([mongo rootname] -; (let [{:keys [math-schema-date math-env]} (:config mongo) -; math-env (or math-env :dev)] -; (str rootname "_" (name math-env) "_" math-schema-date))) -; ([mongo rootname basename] (str (collection-name mongo rootname) "_" basename))) +;; The following functions use math_env in SQL queries as a database field +;; This is a legacy database schema field that is kept for compatibility with "prod" environment (defn poll-tasks [component last-timestamp] (->> - (query - component - (sql/format - {:select [:*] - :from [:worker_tasks] - :where [:and - [:> :created last-timestamp] - [:= :math_env (-> component :config :math-env-string)] - [:= :finished_time nil]]})) - (map (fn [task-record] - (-> task-record - (update :task_type keyword) - (update :task_data (comp #(cheshire/parse-string % true) #(.toString %)))))))) + (query + component + (sql/format + {:select [:*] + :from [:worker_tasks] + :where [:and + [:> :created last-timestamp] + [:= :math_env "prod"] + [:= :finished_time nil]]})) + (map (fn [task-record] + (-> task-record + (update :task_type keyword) + (update :task_data (comp #(cheshire/parse-string % true) #(.toString %)))))))) (defn zid-from-rid [rid] @@ -278,13 +273,13 @@ (defn ptpt-counts [postgres] (query - postgres - {:select [:*] - :from [[{:select [:zid [:%count-distinct.pid :ptpt_cnt]] - :from [:votes] - :group-by [:zid]} - :counts]] - :where [:> :counts.ptpt_cnt 5]})) + postgres + {:select [:*] + :from [[{:select [:zid [:%count-distinct.pid :ptpt_cnt]] + :from [:votes] + :group-by [:zid]} + :counts]] + :where [:> :counts.ptpt_cnt 5]})) (defn query-zid-from-rid [component rid] (query component (zid-from-rid rid))) @@ -292,50 +287,39 @@ (defn inc-math-tick [postgres zid] (log/info "inc-math-tick" zid) - (:math_tick (first (query postgres ["insert into math_ticks (zid, math_env) values (?, ?) on conflict (zid, math_env) do update set modified = now_as_millis(), math_tick = (math_ticks.math_tick + 1) returning math_tick;" zid (-> postgres :config :math-env-string)])))) + (:math_tick (first (query postgres ["insert into math_ticks (zid, math_env) values (?, ?) on conflict (zid, math_env) do update set modified = now_as_millis(), math_tick = (math_ticks.math_tick + 1) returning math_tick;" zid "prod"])))) (defn pg-json [data] (doto (PGobject.) - (.setType "json") - (.setValue (cheshire/encode data)))) + (.setType "json") + (.setValue (cheshire/encode data)))) (defn insert-correlationmatrix! [postgres rid math-tick data] - (query postgres ["insert into math_report_correlationmatrix (rid, math_env, math_tick, data) values (?,?,?,?) on conflict (rid, math_env) do update set data = excluded.data, math_tick = excluded.math_tick returning rid;" rid (-> postgres :config :math-env-string) math-tick (pg-json data)])) - - -;; TODO Fix this; need task-type in here as well for this to work -;(defn mark-task-complete! -; [postgres task-type task-bucket] -; (log/info "mark-task-complete called for task-type, task-bucket:" task-type task-bucket) -; (jdbc/update! -; (:db-spec postgres) -; :worker_tasks -; {:finished_time (System/currentTimeMillis)} -; ["task_type = ? and task_bucket = ?" task-type task-bucket])) + (query postgres ["insert into math_report_correlationmatrix (rid, math_env, math_tick, data) values (?,?,?,?) on conflict (rid, math_env) do update set data = excluded.data, math_tick = excluded.math_tick returning rid;" rid "prod" math-tick (pg-json data)])) + + ;; Marks all tasks with the same task_bucket as done. (defn mark-task-complete! [postgres task_type task_bucket] (log/info "mark-task-complete" task_bucket) - (query postgres ["update worker_tasks set finished_time = now_as_millis() where math_env = (?) and task_type = (?) and task_bucket = (?) returning finished_time;" (-> postgres :config :math-env-string) task_type task_bucket])) + (query postgres ["update worker_tasks set finished_time = now_as_millis() where math_env = (?) and task_type = (?) and task_bucket = (?) returning finished_time;" "prod" task_type task_bucket])) (defn upload-math-main [postgres zid math-tick data] (log/info "upload-math-main for zid" zid) - (let [math-env (-> postgres :config :math-env-string)] - (query postgres - ["insert into math_main (zid, math_env, last_vote_timestamp, math_tick, data, caching_tick) - values (?,?,?,?,?, COALESCE((select max(caching_tick) + 1 from math_main where math_env = (?)), 1)) - on conflict (zid, math_env) - do update set modified = now_as_millis(), - data = excluded.data, - last_vote_timestamp = excluded.last_vote_timestamp, - math_tick = excluded.math_tick, - caching_tick = excluded.caching_tick - returning zid;" - ;; I believe math env is twice here because it gets used in two separate ? - zid math-env (:lastVoteTimestamp data) math-tick (pg-json data) math-env]))) + (query postgres + ["insert into math_main (zid, math_env, last_vote_timestamp, math_tick, data, caching_tick) + values (?,?,?,?,?, COALESCE((select max(caching_tick) + 1 from math_main where math_env = (?)), 1)) + on conflict (zid, math_env) + do update set modified = now_as_millis(), + data = excluded.data, + last_vote_timestamp = excluded.last_vote_timestamp, + math_tick = excluded.math_tick, + caching_tick = excluded.caching_tick + returning zid;" + zid "prod" (:lastVoteTimestamp data) math-tick (pg-json data) "prod"])) (defn upload-math-profile [postgres zid data] @@ -345,7 +329,7 @@ values (?,?,?) on conflict (zid, math_env) do update set modified = now_as_millis(), data = excluded.data returning zid;" - zid (-> postgres :config :math-env-string) (pg-json data)])) + zid "prod" (pg-json data)])) (defn upload-math-ptptstats [postgres zid math-tick data] @@ -358,13 +342,7 @@ data = excluded.data, math_tick = excluded.math_tick returning zid;" - zid (-> postgres :config :math-env-string) math-tick (pg-json data)])) - -;; XXX Not using this anywhere apparently so should remove -;(defn upload-math-cache -; [postgres zid data] -; (log/info "upload-math-cache for zid" zid) -; (query postgres ["insert into math_cache (zid, math_env, data) values (?,?,?) on conflict (zid, math_env) do update set modified = now_as_millis(), data = excluded.data returning zid;" zid (name (-> postgres :config :math-env)) (pg-json data)])) + zid "prod" math-tick (pg-json data)])) (defn upload-math-bidtopid [postgres zid math-tick data] @@ -377,25 +355,25 @@ data = excluded.data, math_tick = excluded.math_tick returning zid;" - zid (-> postgres :config :math-env-string) math-tick (pg-json data)])) + zid "prod" math-tick (pg-json data)])) (defn upload-math-exportstatus [postgres zid filename data] {:pre [postgres zid filename data]} (log/info "upload-math-exportstatus for zid" zid) (query - postgres - ["insert into math_exportstatus (zid, math_env, filename, data, modified) + postgres + ["insert into math_exportstatus (zid, math_env, filename, data, modified) values (?,?,?,?, now_as_millis()) on conflict (zid, math_env) do update set modified = now_as_millis(), data = excluded.data, filename = excluded.filename returning zid;" - zid - (-> postgres :config :math-env-string) - filename - (pg-json data)])) + zid + "prod" + filename + (pg-json data)])) (defn decode-pg-json @@ -406,14 +384,14 @@ [postgres zid filename] (log/info "get-math-exportstatus for zid" zid) (->> - (query postgres ["select * from math_exportstatus where zid = (?) and math_env = (?) and filename = (?);" zid (-> postgres :config :math-env-string) filename]) - first - :data - decode-pg-json)) + (query postgres ["select * from math_exportstatus where zid = (?) and math_env = (?) and filename = (?);" zid "prod" filename]) + first + :data + decode-pg-json)) (defn get-math-tick [postgres zid] - (:math_tick (first (query postgres ["select math_tick from math_ticks where zid = (?) and math_env = (?);" zid (-> postgres :config :math-env-string)])))) + (:math_tick (first (query postgres ["select math_tick from math_ticks where zid = (?) and math_env = (?);" zid "prod"])))) (defn load-conv @@ -421,23 +399,18 @@ as found in the :repness" [postgres zid] (log/info "load-conv called for zid" zid) - (let [row (first (query postgres ["select * from math_main where zid = (?) and math_env = (?);" zid (-> postgres :config :math-env-string)]))] + (let [row (first (query postgres ["select * from math_main where zid = (?) and math_env = (?);" zid "prod"]))] (if row ;; TODO Make sure this loads with keywords for map keys, except where they should be integers (cheshire/parse-string - (.toString (:data row)) - (fn [x] - (try - (Long/parseLong x) - (catch Exception _ - (keyword x))))) + (.toString (:data row)) + (fn [x] + (try + (Long/parseLong x) + (catch Exception _ + (keyword x))))) row))) - ; (mc/find-one-as-map - ; (:db mongo) - ; (math-collection-name mongo "main") - ; {:zid zid})) - (comment (require '[polismath.runner :as runner]) @@ -454,14 +427,14 @@ ;(query postgres ["insert into math_ticks (zid) values (?) on conflict (zid) do update set modified = now_as_millis(), math_tick = (math_ticks.math_tick + 1) returning *;" 12480]) (poll-tasks postgres 0) (query - postgres - (-> (honey/update :worker_tasks) - (honey/values [{}]))) + postgres + (-> (honey/update :worker_tasks) + (honey/values [{}]))) (jdbc/execute! - (:db-spec postgres) - (-> (honey/update :worker_tasks) - (honey/value))) + (:db-spec postgres) + (-> (honey/update :worker_tasks) + (honey/value))) (try (mark-task-complete! postgres :task-type 1) ; Fixed: added missing task-type parameter @@ -469,8 +442,8 @@ (query - postgres - (report-tids 1)) + postgres + (report-tids 1)) :endcomment) :ok diff --git a/math/src/polismath/components/random.clj b/math/src/polismath/components/random.clj deleted file mode 100644 index b781fbccf7..0000000000 --- a/math/src/polismath/components/random.clj +++ /dev/null @@ -1,7 +0,0 @@ -;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . - -(ns polismath.components.random - "This will eventually house a random number generator component which assists in the tracking of seed state") - - - diff --git a/math/src/polismath/components/server.clj b/math/src/polismath/components/server.clj deleted file mode 100644 index 7ffa57793f..0000000000 --- a/math/src/polismath/components/server.clj +++ /dev/null @@ -1,72 +0,0 @@ -;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . - -(ns polismath.components.server - (:require - ;; XXX Deprecate; use component config directly - [clojure.core.async :as async :refer [chan >!! ! > Starting server component with config:" (:server config)) - (let [wrapped-handler (wrap-handler (:handler app)) - jetty-server (jetty/run-jetty wrapped-handler - (merge {:join? false :port 8080} - (:server config) - opts))] - (.start jetty-server) - (assoc component :jetty-server jetty-server))) - (stop [component] - (log/info "<< Stopping server component") - (try - (.stop jetty-server) - (catch Exception e - (log/error e "Unable to stop server!"))) - component)) - - -(defn create-server - ;; Default handler just return the healthcheck response always - ([opts] - (let [opts (merge {:app {:handler healthcheck-handler}} - opts)] - (map->Server opts))) - ([] - (create-server {}))) diff --git a/math/src/polismath/conv_man.clj b/math/src/polismath/conv_man.clj index 8183a7ed3c..3fc3a8fcde 100644 --- a/math/src/polismath/conv_man.clj +++ b/math/src/polismath/conv_man.clj @@ -2,22 +2,20 @@ (ns polismath.conv-man "This is the namespace for the conversation manager" - (:require [polismath.math.named-matrix :as nm] - [polismath.math.conversation :as conv] - [polismath.math.clusters :as clust] - [polismath.meta.metrics :as met] - [polismath.meta.notify :as notify] - [polismath.components.env :as env] - [polismath.components.postgres :as db] - [polismath.math.corr :as corr] - [polismath.utils :as utils] - [clojure.core.matrix.impl.ndarray] - [clojure.core.async :as async :refer [go go-loop ! !! alts!! alts! chan dropping-buffer put! take!]] - [taoensso.timbre :as log] - [com.stuartsierra.component :as component] - [plumbing.core :as pc] - [schema.core :as s] - [polismath.components.postgres :as postgres])) + (:require + [clojure.core.async :as async :refer [>! >!! alts! chan go go-loop]] + [clojure.core.matrix.impl.ndarray] + [com.stuartsierra.component :as component] + [plumbing.core :as pc] + [polismath.components.postgres :as db] + [polismath.math.clusters :as clust] + [polismath.math.conversation :as conv] + [polismath.math.named-matrix :as nm] + [polismath.meta.metrics :as met] + [polismath.meta.notify :as notify] + [polismath.utils :as utils] + [schema.core :as s] + [taoensso.timbre :as log])) @@ -72,16 +70,15 @@ :subgroup-votes :subgroup-repness :comment-priorities}))) - ;:subgroup-ptpt-stats}))) (defn columnize ([stats keys] (->> keys (map - (fn [k] - (let [vals (map (fn [stat] (get stat k)) stats)] - [k vals]))) + (fn [k] + (let [vals (map (fn [stat] (get stat k)) stats)] + [k vals]))) (into {}))) ([stats] (let [keys (-> stats first keys)] @@ -96,8 +93,8 @@ (defn handle-profile-data "For now, just log profile data. Eventually want to send to influxDB and graphite." - [{:as conv-man :keys [postgres config]} conv & {:keys [recompute n-votes finish-time] :as extra-data}] - (if-let [prof-atom (:profile-data conv)] + [{:keys [postgres]} conv & {:as extra-data}] + (when-let [prof-atom (:profile-data conv)] (let [prof @prof-atom tot (apply + (map second prof)) prof (assoc prof :total tot)] @@ -129,9 +126,7 @@ "This function is what actually gets sent to the conv-manager. In addition to the conversation and vote batches up in the channel, we also take an error-callback. Eventually we'll want to pass opts through here as well." [conv-man conv votes] - (let [start-time (System/currentTimeMillis) - config (:config conv-man) - pg (:postgres conv-man)] + (let [start-time (System/currentTimeMillis)] (log/info "Starting conversation update for zid:" (:zid conv)) ;; Need to expose opts for conv-update through config... XXX (let [updated-conv (conv/conv-update conv votes) @@ -156,7 +151,7 @@ updated-conv))) (defn write-conv-updates! - [{:as conv-man :keys [postgres]} {:as updated-conv :keys [zid]} math-tick] + [{:keys [postgres]} {:as updated-conv :keys [zid]} math-tick] ;; TODO Really need to extract these writes so that mod updates do whta they're supposed to! And also run in async/thread for better parallelism ; Format and upload main results (async/thread @@ -202,21 +197,13 @@ (nm/update-nmat (->> (db/conv-poll (:postgres conv-man) zid 0) (map (fn [vote-row] (mapv (partial get vote-row) [:pid :tid :vote]))))))) (conv/mod-update - (db/conv-mod-poll (:postgres conv-man) zid 0))) + (db/conv-mod-poll (:postgres conv-man) zid 0))) ; would be nice to have :recompute :initial (assoc (conv/new-conv) :zid zid :recompute :full))) (defn generate-report-data! - [{:as conv-man :keys [postgres]} conv math-tick report-data] + [_conv-man _conv _math-tick report-data] (log/error "Report generation requested; No longer supported:" report-data)) - ;(log/info "Generating report data for report:" report-data)) - ;(let [rid (:rid report-data) - ;tids (map :tid (postgres/query (:postgres conv-man) (postgres/report-tids rid))) - ;corr-mat (corr/compute-corr conv tids)] - ;(async/thread - ;(postgres/insert-correlationmatrix! postgres rid math-tick corr-mat) - ;;; TODO update to submit usng task type and task bucket - ;(postgres/mark-task-complete! postgres "generate_report_data" rid)))) @@ -234,16 +221,6 @@ acc#)))) -(defn take-all!! [c] - "Given a channel, takes all values currently in channel and places in a vector. Must be called - within a go block." - (loop [acc []] - (let [[v c] (alts!! [c] :default nil)] - (if (not= c :default) - (recur (conj acc v)) - acc)))) - - (defn split-batches "This function splits message batches as sent to conv actor up by the first item in batch vector (:votes :moderation) so messages can get processed properly" @@ -251,10 +228,10 @@ (->> messages (group-by :message-type) (pc/map-vals - (fn [labeled-batches] - (->> labeled-batches - (map :message-batch) - (flatten)))))) + (fn [labeled-batches] + (->> labeled-batches + (map :message-batch) + (flatten)))))) @@ -264,19 +241,19 @@ ;; Here's the multimethod at the core of the messages the conv actor may act upon. (defmulti react-to-messages - (fn [conv-man conv message-type messages] - message-type)) + (fn [_conv-man _conv message-type _messages] + message-type)) (defmethod react-to-messages :votes - [conv-man conv _ messages] + [conv-man conv _message-type messages] (conv-update conv-man conv messages)) (defmethod react-to-messages :moderation - [conv-man conv _ messages] + [_conv-man conv _message-type messages] (conv/mod-update conv messages)) (defmethod react-to-messages :generate_report_data - [conv-man conv _ messages] + [conv-man conv _message-type messages] (let [math-tick (or (:math-tick conv) (:math_tick conv))] (doseq [report-task messages] (try @@ -293,13 +270,10 @@ (let [zid (:zid conv-actor) zid-str (str "zid=" zid) retry-chan (:retry-chan conv-actor) - notify-message (str "Failed conversation update on " (-> conv-man :config :math-env) " for message-type " message-type " and " zid-str)] - (try - (let [stack-trace (notify/error-message-body update-error)] - (notify/notify-team (:config conv-man) (str "Polismath conv-man error: " message-type) zid notify-message stack-trace)) - (catch Exception e - (log/error e "Unable to notify team"))) - (log/error update-error notify-message) + notify-message (str "Failed conversation update for message-type " message-type " and " zid-str)] + (let [stack-trace (notify/error-message-body update-error)] + (log/error update-error notify-message) + (log/error "Stack trace:" stack-trace)) (.printStackTrace update-error) ; Try requeing the votes that failed so that if we get more, they'll get replayed (try @@ -314,12 +288,12 @@ duration (- end start-time)] ;; Update to use MetricSender component XXX (met/send-metric (:metrics conv-man) "math.pca.compute.fail" duration)) - (catch Exception e + (catch Exception _e (log/error "Unable to send metrics for failed compute for" zid-str))) ; Try to save conversation state for debugging purposes (try (conv/conv-update-dump conv messages update-error) - (catch Exception e + (catch Exception _e (log/error "Unable to perform conv-update dump for" zid-str))))) @@ -328,14 +302,14 @@ (defn react-to-messages! [conv-man conv-actor message-type messages] (let [start-time (System/currentTimeMillis) - {:keys [zid conv retry-chan]} conv-actor + {:keys [zid conv]} conv-actor update-fn (fn [conv'] (try (if-let [updated-conv (react-to-messages conv-man conv' message-type messages)] (do (let [math-tick (or (:math-tick updated-conv) ;; pass through for report generation - (postgres/inc-math-tick (:postgres conv-man) zid))] + (db/inc-math-tick (:postgres conv-man) zid))] (write-conv-updates! conv-man updated-conv math-tick)) updated-conv) ;; if nil, don't update, for just side effects @@ -350,8 +324,8 @@ (defn go-act! [conv-man conv-actor] - (let [{:keys [kill-chan conversations]} conv-man - {:keys [zid conv message-chan retry-chan]} conv-actor] + (let [{:keys [kill-chan]} conv-man + {:keys [message-chan retry-chan]} conv-actor] (go-loop [] ;; If nil comes through as the first message, then the chan is closed, and we should be done, not continue looping forever (let [[first-msg c] (async/alts! [kill-chan message-chan] :priority true)] @@ -373,7 +347,7 @@ ;; Put the actor together and (defn conv-actor - [{:as conv-man :keys [conversations kill-chan config]} zid] + [{:as conv-man :keys [config]} zid] (log/info "Starting message batch queue and handler routine for conv zid:" zid) (let [conv (load-or-init conv-man zid :recompute (:recompute config)) _ (log/info "Conversation loaded for conv zid:" zid) @@ -432,7 +406,7 @@ ;; Close all our message channels for good measure (log/debug "conversations:" conversations) (go (>! kill-chan :kill)) - (doseq [[zid {:keys [message-chan]}] @conversations] + (doseq [[_ {:keys [message-chan]}] @conversations] (async/close! message-chan)) ;; Not sure, but we might want this for GC (reset! conversations nil) @@ -450,9 +424,9 @@ ;; Need to think about what to do if failed conversations lead to messages piling up in the message queue XXX (defn queue-message-batch! "Queue message batches for a given conversation by zid" - [{:as conv-man :keys [conversations config kill-chan]} message-type zid message-batch] + [{:as conv-man :keys [conversations kill-chan]} message-type zid message-batch] (when-not (async/poll! kill-chan) - (if-let [{:keys [conv message-chan]} (get @conversations zid)] + (if-let [{:keys [message-chan]} (get @conversations zid)] ;; Then we already have a go loop running for this (>!! message-chan {:message-type message-type :message-batch message-batch}) ;; Then we need to initialize the conversation and set up the conversation channel and go routine diff --git a/math/src/polismath/darwin/core.clj b/math/src/polismath/darwin/core.clj index 368f0c1a89..636d6996b9 100644 --- a/math/src/polismath/darwin/core.clj +++ b/math/src/polismath/darwin/core.clj @@ -2,115 +2,50 @@ (ns polismath.darwin.core "The darwin.core namespace wraps the more pure (+ writing files/zips) code in darwin.export. Deals with parsing the - params, updating the db status, uploading to aws, etc. This gets hooked up in the tasks namespace." + params, updating the db status, etc. This gets hooked up in the tasks namespace." (:require - [polismath.darwin.export :as export] - ;; XXX Deprecate; use component config directly - ;[polismath.components.env :as env] - [polismath.components.postgres :as db] - [clojure.core.async :as async :refer [chan >!! ! darwin :config :aws)) - -(defn full-aws-path - [darwin filename] - (str (-> darwin :config :math-env-string) "/" filename)) - -(defn upload-to-aws - [{:as darwin :keys [s3-client]} filename] - (aws/invoke s3-client - {:op :PutObject - :request {:Bucket "polis-datadump" - :Body (io/input-stream (io/file filename)) - :Key (full-aws-path darwin filename)}})) - -(defn send-email-notification-via-polis-api! - [darwin {:as params :keys [zinvite email filename]}] - (try - (let [darwin-config (-> darwin :config) - response - ;; The next three lines should probably be extracted - (client/post (str (:webserver-url darwin-config) "/sendEmailExportReady") - {:form-params {:webserver_username (:webserver-username darwin-config) - :webserver_pass (:webserver-pass darwin-config) - :email email - :filename filename - :raise false - :conversation_id zinvite} - :content-type :json - :throw-entire-message? true})] - (log/info "send email notification response:\n" (with-out-str (clojure.pprint/pprint response)))) - (catch Exception e - (log/error e "failed to send email:\n")))) - - (defn handle-completion! [{:as darwin :keys [postgres]} {:as params :keys [filename task_bucket]}] (log/info "Completed export computation for filename" filename "params:" (with-out-str (str params))) - (upload-to-aws darwin filename) - ;;(upload-to-polis-polismath.darwin.core darwin filename) (notify-of-status darwin params "complete") - (postgres/mark-task-complete! postgres "generate_export_data" task_bucket) - (when (:email params) (send-email-notification-via-polis-api! darwin params))) + (db/mark-task-complete! postgres "generate_export_data" task_bucket)) -;; The filename is actually pretty iportant. -;; It should be unique between different exports, as is used as the identifying key for the aws buckets, postgres -;; tracking and as part of the API requests. +;; The filename is actually pretty important. +;; It should be unique between different exports, as it is used for postgres tracking. (defn generate-filename "Generates a filename based on request-params" - [{:as request-params :keys [zinvite at-time format]}] + [{:keys [zinvite at-time format]}] {:pre [zinvite format]} (let [last-updated (or at-time (System/currentTimeMillis)) - ext (case format :excel "xlsx" :csv "zip") + ext (case format :csv "zip") filename (str "polis-export-" zinvite "-" last-updated "." ext)] filename)) @@ -125,15 +60,12 @@ (if (number? x) (long x) (try (Long/parseLong x) - (catch Exception e nil)))) - -(defn- between? [a b x] - (and x (< a x) (> b x))) + (catch Exception _e nil)))) (defn- parse-and-validate-timeout [x] (let [x (try (Long/parseLong x) - (catch Exception e (throw (Exception. "Invalid timeout value"))))] + (catch Exception _e (throw (Exception. "Invalid timeout value"))))] (assert (and x (< 0 x) (>= 29000 x)) "Invalid timout value") x)) @@ -146,21 +78,21 @@ (defn params-with-zid [darwin params] (assoc params - :zid (or (:zid params) - (db/get-zid-from-zinvite (:postgres darwin) (:zinvite params))))) + :zid (or (:zid params) + (db/get-zid-from-zinvite (:postgres darwin) (:zinvite params))))) (defn params-with-filename [params] (assoc params - :filename - (or (:filename params) - (generate-filename params)))) + :filename + (or (:filename params) + (generate-filename params)))) (defn params-with-zinvite [darwin params] (assoc params - :zinvite (or (:zinvite params) - (db/get-zinvite-from-zid (:postgres darwin) (:zid params))))) + :zinvite (or (:zinvite params) + (db/get-zinvite-from-zid (:postgres darwin) (:zid params))))) (defn parsed-params @@ -168,68 +100,28 @@ [darwin params] ;(log/info "Here are the params:" params) (->> - params - (reduce - (fn [m [k v]] - ;; Don't really need this if we have params instead of query params, but whateves - (let [k (keyword k)] - (assoc m k ((or (parsers k) identity) v)))) - {}) - (params-with-zid darwin) - (params-with-zinvite darwin) - (params-with-filename))) - - - - - -(comment - ;; Here we're putting everything together - ;; This may not all be 100% correct, as it was copied over from the repl... but I ran through all but the spit and sanity checks pass - (require '[polismath.runner :as runner] - '[polismath.system :as system]) - ;(runner/run! system/base-system {:math-env :preprod}) - ;; Load the data for 15117 (zinvite 2ez5beswtc) - ;(def focus-id 15117) - (def focus-zinvite "36jajfnhhn") - (def focus-id 15228) - (def conv (conv-man/load-or-init (:conversation-manager runner/system) focus-id)) - ;conv - (matrix/shape (nm/get-matrix (:rating-mat conv))) - ;; Compute hclust on a cleaned, transposed rating matrix - (time - (def hclusters (hclust (cleaned-rating-mat-transpose conv)))) - (count (-> hclusters first :members)) - ;; Compute corr matrix - (time - (def corr-mat (correlation-matrix conv))) - (keys corr-mat) - (matrix/shape (:matrix corr-mat)) - (def corr-mat' (blockify-corr-matrix corr-mat hclusters)) - (keys corr-mat') - (count (:comments corr-mat')) - ;; Spit this out - (spit-matrix (str focus-zinvite ".corrmat.json") corr-mat) - (spit-matrix (str focus-zinvite ".corrmat-whclust.json") corr-mat') - (spit-hclust (str focus-zinvite ".hclust.json") hclusters) - ;; All done - :endcomment) + params + (reduce + (fn [m [k v]] + ;; Don't really need this if we have params instead of query params, but whateves + (let [k (keyword k)] + (assoc m k ((or (parsers k) identity) v)))) + {}) + (params-with-zid darwin) + (params-with-zinvite darwin) + (params-with-filename))) + ;; Route everything together, build handlers, etc -;; NOOP right now for compatibility -(defrecord Darwin [config postgres conversation-manager s3-client] +(defrecord Darwin [config postgres conversation-manager] component/Lifecycle (start [component] - (if-let [aws-config (get config :aws)] - (let [creds (aws-creds/basic-credentials-provider aws-config)] - (assoc component - :s3-client (aws/client {:api :s3 - :credentials-provider creds}))) - (do (log/info "Skipping AWS connection") - component))) + (log/info ">> Starting darwin component") + component) (stop [component] + (log/info "<< Stopping darwin component") component)) diff --git a/math/src/polismath/darwin/export.clj b/math/src/polismath/darwin/export.clj index e01e5adff4..751afb029c 100644 --- a/math/src/polismath/darwin/export.clj +++ b/math/src/polismath/darwin/export.clj @@ -1,42 +1,20 @@ ;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . (ns polismath.darwin.export - (:require [clojure.java.io :as io] - [polismath.components.postgres :as db] - ;; Probably should move the function we need out, since this should be meta only XXX - [polismath.meta.microscope :as micro] - [polismath.math.clusters :as clust] - [polismath.math.named-matrix :as nm] - [polismath.math.conversation :as conv] - [polismath.utils :as utils] - [clojure.math.numeric-tower :as math] - [clojure.core.matrix :as mat] - [clj-time.core :as t] - [clj-time.coerce :as co] - ;; Think I'm going to use the second one here, since it's simpler (less mutable) - ;[dk.ative.docjure.spreadsheet :as spreadsheet] - ;[clj-excel.core :as excel] - [semantic-csv.core :as scsv] - [clojure-csv.core :as csv] - [clojure.pprint :refer [pprint]] - [clojure.core.matrix :as mat] - [clojure.tools.trace :as tr] - [taoensso.timbre :as log] - [clojure.tools.cli :refer [parse-opts]] - [polismath.components.postgres :as postgres] - [polismath.conv-man :as conv-man] - [plumbing.core :as plmb]) - (:import [java.util.zip ZipOutputStream ZipEntry])) - -;(mat/set-current-implementation :vectorz) - -;; Here's rougly what we want for data export. We have the following sheets per conversation. -;; -;; Summary: -;; * N Views -;; * N Voters -;; * N Voters "in conv" -;; * N Commenters + (:require + [clojure-csv.core :as csv] + [clojure.java.io :as io] + [clojure.string :as string] + [plumbing.core :as plmb] + [polismath.components.postgres :as db] + [polismath.conv-man :as conv-man] + [polismath.math.conversation :as conv] + [polismath.math.named-matrix :as nm] + [polismath.utils :as utils] + [semantic-csv.core :as scsv] + [taoensso.timbre :as log]) + (:import + [java.util.zip ZipEntry ZipOutputStream])) ;; * N Groups ;; * url ;; @@ -62,30 +40,26 @@ "/" filename)) - -(defn db-spec [darwin] - (-> darwin :postgres :db-spec)) - ;; Database calls for various things (defn get-zids-for-uid [darwin uid] (map :zid - (db/query (:postgres darwin) - {:select [:*] - :from [:conversations] - :where [:= :owner uid]}))) + (db/query (:postgres darwin) + {:select [:*] + :from [:conversations] + :where [:= :owner uid]}))) (defn get-zinvite-from-zid [darwin zid] (-> - (db/query (:postgres darwin) - {:select [:zid :zinvite] - :from [:zinvites] - :where [:= :zid zid]}) - first - :zinvite)) + (db/query (:postgres darwin) + {:select [:zid :zinvite] + :from [:zinvites] + :where [:= :zid zid]}) + first + :zinvite)) (defn get-conversation-votes* @@ -100,7 +74,7 @@ {:select [:*] :from [:votes] :where [:and [:= :zid zid] - [:<= :created final-vote-timestamp]] + [:<= :created final-vote-timestamp]] :order-by [:zid :tid :pid :created]}))) (defn get-corrected-conversation-votes @@ -120,11 +94,11 @@ "Return a map with :topic and :description keys" [darwin zid] (-> - (db/query (:postgres darwin) - {:select [:zid :topic :description :created] - :from [:conversations] - :where [:= :zid zid]}) - first)) + (db/query (:postgres darwin) + {:select [:zid :topic :description :created] + :from [:conversations] + :where [:= :zid zid]}) + first)) (defn get-participation-data* ([darwin zid] @@ -137,7 +111,7 @@ {:select [:zid :pid :vote_count :created] :from [:participants] :where [:and [:= :zid zid] - [:<= :created final-timestamp]]}))) + [:<= :created final-timestamp]]}))) (defn get-participation-data [& args] @@ -180,7 +154,7 @@ "Takes in rating matrix and set of all votes, and computes the summary stats for the conversation" [darwin - {:keys [n n-cmts group-clusters base-clusters zid rating-mat] :as conv} + {:keys [n n-cmts group-clusters zid]} votes comments-data participants] @@ -190,7 +164,7 @@ ;; Do anything needed with the data to prep {:keys [topic description]} @conv-data url (str "https://pol.is/" @zinvite)] - ;; Return the table of stuff to go into excel + ;; Return the table of stuff {:topic topic :url url :n-views (count-distinct-col :pid participants) @@ -202,7 +176,7 @@ :description description})) (defn render-summary-with - "Intended for formatting the summary data's psuedo-headers for either excel or csv. Takes the + "Intended for formatting the summary data's psuedo-headers for csv. Takes the data as produced by summary data, and a key-mapping collection of keys as in the data to header names to output, returning a collection of rows to be spit out." [data key-mapping] @@ -218,13 +192,13 @@ [votes participants comments] (sort-by :created (mapcat - (fn [collection tag] - (map #(assoc % :tag tag) collection)) - [votes participants comments] - [:vote :participant :comment]))) + (fn [collection tag] + (map #(assoc % :tag tag) collection)) + [votes participants comments] + [:vote :participant :comment]))) (defmulti update-history* - (fn [last-history datom] (:tag datom))) + (fn [_last-history datom] (:tag datom))) (defn update-history "For stats-history; takes the last history value and updates it with the given record. Delegates @@ -233,8 +207,8 @@ (conj history (-> datom (->> (update-history* (or (last history) - {:n-votes 0 :n-comments 0 :n-visitors 0 :n-voters 0 :n-commenters 0 :ctxt {:voters #{} - :commenters #{}}}))) + {:n-votes 0 :n-comments 0 :n-visitors 0 :n-voters 0 :n-commenters 0 :ctxt {:voters #{} + :commenters #{}}}))) (assoc :time (:created datom))))) (defmethod update-history* :vote @@ -248,7 +222,7 @@ (update-in [:ctxt :voters] conj (:pid datom)))) (defmethod update-history* :participant - [last-history datom] + [last-history _datom] (-> last-history (update-in [:n-visitors] inc))) @@ -287,69 +261,63 @@ "Takes group clusters and base clusters and flattens them out into a cluster mapping to ptpt ids directly" [group-clusters base-clusters] (map - (fn [gc] - (update-in gc - [:members] - (fn [members] - (mapcat - (fn [bid] - ;; get the base cluster, then get it's members, mapcat them (a level up) - (:members (ffilter #(= (:id %) bid) base-clusters))) - members)))) - group-clusters)) + (fn [gc] + (update-in gc + [:members] + (fn [members] + (mapcat + (fn [bid] + ;; get the base cluster, then get it's members, mapcat them (a level up) + (:members (ffilter #(= (:id %) bid) base-clusters))) + members)))) + group-clusters)) (defn participant-xids [darwin conv] (->> - (db/query - (:postgres darwin) - {:select [:participants.pid :xids.uid :xid] - :from [:xids] - :join [:conversations [:= :conversations.owner :xids.owner] - :participants [:= :participants.uid :xids.uid]] - :where [:and - [:= :conversations.zid (:zid conv)] - [:= :participants.zid (:zid conv)]]}) - (map (fn [{:keys [pid xid]}] - [pid xid])) - (into {}))) + (db/query + (:postgres darwin) + {:select [:participants.pid :xids.uid :xid] + :from [:xids] + :join [:conversations [:= :conversations.owner :xids.owner] + :participants [:= :participants.uid :xids.uid]] + :where [:and + [:= :conversations.zid (:zid conv)] + [:= :participants.zid (:zid conv)]]}) + (map (fn [{:keys [pid xid]}] + [pid xid])) + (into {}))) ;; participant-id, group-id, n-votes, n-comments, n-aggre, n-disagree, (defn participants-votes-table - [darwin conv votes comments {:as kw-args :keys [include-xid]}] + [darwin conv votes comments {:keys [include-xid]}] (let [mat (reconstruct-vote-matrix votes) flattened-clusters (flatten-clusters (:group-clusters conv) (:base-clusters conv)) xids (participant-xids darwin conv)] (concat - ;; The header - [(concat ["participant"] - (when include-xid ["xid"]) - ["group-id" "n-comments" "n-votes" "n-agree" "n-disagree"] - (nm/colnames mat))] - ;; The rest of the data - (map - (fn [ptpt row] - (concat [ptpt] - (when include-xid [(get xids ptpt)]) - [(:id (ffilter #(some #{ptpt} (:members %)) flattened-clusters)) - (count (filter #(= (:pid %) ptpt) comments)) - (count (remove nil? row)) - ;; XXX God damn aggree vs disagree... - ;; Fixed this upstream, for now; so should be good to go once we've fixed it at the source. But - ;; keep an eye on it for now... XXX - (count (filter #{1} row)) - (count (filter #{-1} row))] - row)) - (nm/rownames mat) - (.matrix mat))))) - -(defn format-vote-matrix-header - "Apply format-header function to each element of header in a collection of row vectors" - [data format-header] - (concat [(mapv format-header (first data))] - (rest data))) + ;; The header + [(concat ["participant"] + (when include-xid ["xid"]) + ["group-id" "n-comments" "n-votes" "n-agree" "n-disagree"] + (nm/colnames mat))] + ;; The rest of the data + (map + (fn [ptpt row] + (concat [ptpt] + (when include-xid [(get xids ptpt)]) + [(:id (ffilter #(some #{ptpt} (:members %)) flattened-clusters)) + (count (filter #(= (:pid %) ptpt) comments)) + (count (remove nil? row)) + ;; XXX God damn aggree vs disagree... + ;; Fixed this upstream, for now; so should be good to go once we've fixed it at the source. But + ;; keep an eye on it for now... XXX + (count (filter #{1} row)) + (count (filter #{-1} row))] + row)) + (nm/rownames mat) + (.matrix mat))))) ;; Comments @@ -362,115 +330,21 @@ (assoc comment-data :group-informed-consensus (get group-aware-consensus tid))) comments)) -;(sort (keys theconv)) -;(:subgroup-repness theconv) - -;(:group-aware-consensus theconv) - -;(comment - ;(require '[polismath.runner :as runner]) - ;(runner/run! polismath.system/export-system) - ;(load-conv (:darwin runner/system) :zid 12798) - ;(try - ;(export-conversation (:darwin runner/system) - ;{;;:zid 310273 - ;:zinvite "2demo" - ;:format :csv - ;:filename "2demo.zip" - ;:math-env "prod"}) - ;(catch Exception e - ;(.printStackTrace e)))) - (defn enriched-comments-data "Just adds vote counts to the comments data" [comments votes] (map - (fn [{:keys [tid] :as comment-data}] - (let [comment-votes (filter #(= tid (:tid %)) votes) - ;; Fixed this upstream, for now; so should be good to go once we've fixed it at the source. But - ;; keep an eye on it for now... XXX - aggrees (filter #(= 1 (:vote %)) comment-votes) - disagrees (filter #(= -1 (:vote %)) comment-votes)] - (assoc comment-data - :aggrees (count aggrees) - :disagrees (count disagrees) - :datetime (datetime (:created comment-data))))) - comments)) - - -;; Now some reshaping stuff for excel; mostly just applying headers here -;; Actaul excel things - -(defn stringify-keys - ([m] - (stringify-keys m #(-> % str (clojure.string/replace ":" "")))) - ([m f] - (into {} (map (fn [[k v]] - [(if-not (string? k) (f k) k) v]) - m)))) - - -(defn update-from-map-or-leave - "Little utility for generating a function that either updates based on a map (closed over) or leaves value unchanged" - [m] - (fn [k] - (if-let [v (get m k)] - v - k))) - -(defn excel-format - [export-data] - (-> export-data - (update-in [:summary] - render-summary-with - [[:topic "Topic"] - [:url "URL"] - [:n-views "Views"] - [:n-voters "Voters"] - [:n-voters-in "Voters (in conv)"] - [:n-commenters "Commenters"] - [:n-comments "Comments"] - [:n-groups "Groups"] - [:description "Conversation Description"]]) - (update-in [:stats-history] - (partial scsv/vectorize {:header [:n-votes :n-comments :n-visitors :n-voters :n-commenters] - :format-header {:n-votes "Votes" - :n-visitors "Visitors" - :n-voters "Voters" - :n-comments "Comments" - :n-commenters "Commenters"}})) - (update-in [:comments] - (partial scsv/vectorize {:header [:created :tid :pid :aggrees :disagrees :mod :is_meta :is_seed :group-informed-consensus :txt] - :format-header {:created "Timestamp" - :tid "Comment ID" - :pid "Author" - :aggrees "Aggrees" - :disagrees "Disagrees" - :mod "Moderated" - :is_meta "Metadata" - :is_seed "Seed" - :group-informed-consensus "Group informed consensus" - :txt "Comment body"}})) - (update-in [:votes] - (partial scsv/vectorize {:header [:created :tid :pid :vote] - :format-header {:created "Timestamp" - :tid "Comment ID" - :pid "Voter" - :vote "Vote"}})) - (update-in [:participants-votes] - format-vote-matrix-header - ;; flesh out... - (update-from-map-or-leave {"participant" "Participant" - "group-id" "Group ID" - "n-comments" "Comments" - "n-votes" "Votes" - "n-agree" "Agrees" - "n-disagree" "Disagrees"})) - (stringify-keys (array-map :summary "Summary" - :stats-history "Stats History" - :comments "Comments" - :votes "Votes" - :participants-votes "Participants Votes")))) + (fn [{:keys [tid] :as comment-data}] + (let [comment-votes (filter #(= tid (:tid %)) votes) + ;; Fixed this upstream, for now; so should be good to go once we've fixed it at the source. But + ;; keep an eye on it for now... XXX + aggrees (filter #(= 1 (:vote %)) comment-votes) + disagrees (filter #(= -1 (:vote %)) comment-votes)] + (assoc comment-data + :aggrees (count aggrees) + :disagrees (count disagrees) + :datetime (datetime (:created comment-data))))) + comments)) (defn csv-format @@ -521,12 +395,6 @@ (flush) (.closeEntry zip#))) -(defn move-to-zip-stream - [zip-stream input-filename entry-point] - (with-open [input (io/input-stream input-filename)] - (with-entry zip-stream entry-point - (io/copy input zip-stream)))) - (defn print-csv [table] (->> table @@ -537,9 +405,9 @@ (defn zipfile-basename [filename] (-> filename - (clojure.string/split #"\/") + (string/split #"\/") last - (clojure.string/replace #"\.zip$" ""))) + (string/replace #"\.zip$" ""))) ;; Must assert .zip in filenames or things will break on unzipping XXX @@ -564,20 +432,6 @@ (print-csv (:participants-votes data))))))) -;(defn save-to-excel - ;([filename data] - ;(-> (excel/build-workbook data) - ;(excel/save filename))) - ;;; Should really change both this and the above to use the .zip filename, and take basename for the main dir - ;;; XXX - ;([zip-stream entry-point data] - ;;; Would be nice if we could write directly to the zip stream, but the excel library seems to be doing - ;;; weird things... - ;(let [tmp-file-path (str "tmp/rand-" (rand-int Integer/MAX_VALUE) ".xml")] - ;(save-to-excel tmp-file-path data) - ;(move-to-zip-stream zip-stream tmp-file-path entry-point)))) - - ;; Putting it all together ;; ======================= @@ -585,26 +439,26 @@ [kw] (-> kw (str) - (clojure.string/replace ":" "") + (string/replace ":" "") (Integer/parseInt))) ;; Should be using the conv-man to load the conversation if it's already there (defn load-conv - [darwin & {:keys [zid zinvite] :as args}] - (let [zid (or zid (postgres/get-zid-from-zinvite (:postgres darwin) zinvite))] + [darwin & {:keys [zid zinvite]}] + (let [zid (or zid (db/get-zid-from-zinvite (:postgres darwin) zinvite))] (log/debug "Loading conversation" zid) (-> - (db/load-conv (:postgres darwin) zid) - ;; This should be ok here right? - (conv-man/restructure-json-conv) - (update-in - [:repness] - (partial plmb/map-keys kw->int))))) + (db/load-conv (:postgres darwin) zid) + ;; This should be ok here right? + (conv-man/restructure-json-conv) + (update-in + [:repness] + (partial plmb/map-keys kw->int))))) (defn get-export-data [darwin {:keys [zid zinvite update-math] :as kw-args}] - (let [zid (or zid (postgres/get-zid-from-zinvite (:postgres darwin) zinvite)) + (let [zid (or zid (db/get-zid-from-zinvite (:postgres darwin) zinvite)) ;; assert zid votes (get-corrected-conversation-votes darwin zid) participants (get-participation-data darwin zid) @@ -622,10 +476,10 @@ :comments comments})) (defn get-export-data-at-time - [darwin {:keys [zid zinvite env-overrides at-time update-postgres] :as kw-args}] - (let [zid (or zid (postgres/get-zid-from-zinvite (:postgres darwin) zinvite)) + [darwin {:keys [zid zinvite at-time update-postgres] :as kw-args}] + (let [zid (or zid (db/get-zid-from-zinvite (:postgres darwin) zinvite)) votes (get-corrected-conversation-votes darwin zid at-time) - meta-tids (postgres/get-meta-tids (:postgres darwin) zid) + meta-tids (db/get-meta-tids (:postgres darwin) zid) ;_ (log/info "META TIDS " meta-tids) conv (-> (assoc (conv/new-conv) :zid zid :meta-tids meta-tids) ;; The core math still uses the uncorrected votes @@ -638,7 +492,7 @@ (log/info "Done with conv update") (when update-postgres (log/info "Updating postgres") - (let [math-tick (postgres/inc-math-tick (:postgres darwin) zid)] + (let [math-tick (db/inc-math-tick (:postgres darwin) zid)] (conv-man/write-conv-updates! darwin conv math-tick))) {:votes votes :summary (assoc (summary-data darwin conv votes comments participants) :at-time at-time) @@ -647,34 +501,18 @@ :comments comments})) -;(defn load-from-timestamp - ;[darwin {:keys [zid zivite at-time] :as kw-args}] - ;(let [zid (or zid (postgres/get-zid-from-zinvite (:postgrs darwin) zinvite)) - ;votes (get-conversation-votes darwin zid at-time) - ;math-tick (postgres/inc-math-tick (:postgres darwin) zid) - ;conv (-> (assoc (conv/new-conv) :zid zid) - ;(conv/conv-update votes))] - ;(conv-man/write-conv-updates! {:postgres (:postgres darwin)} - ;conv - ;math-tick) - ;(log/info "Done with conv update"))) - - (defn export-conversation "This is the main API endpoint for the export functionality. Given either :zid or :zinvite, export data to the specified :format and spit results out to :filename. Optionally, a :zip-stream and :entry point may be specified, which can be used for building up items in a zip file. This is used in export/-main to export all convs for a given uid, for example." - ;; Don't forget env-overrides {:math-env "prod"}; should clean up with system - [darwin {:keys [zid zinvite format filename zip-stream writer entry-point env-overrides at-time include-xid] :as kw-args}] + [darwin {:keys [zid zinvite filename zip-stream writer entry-point at-time] :as kw-args}] (log/info "Exporting data for zid =" zid ", zinvite =" zinvite) (log/info "at-time:" at-time) (let [export-data (if at-time (get-export-data-at-time darwin kw-args) (get-export-data darwin kw-args)) - ;; Had to remove excel support because lib outdated vs tech.ml.dataset java dep - ;[formatter saver] (case format :excel [excel-format save-to-excel] :csv [csv-format save-to-csv-zip]) [formatter saver] [csv-format save-to-csv-zip] formatted (formatter export-data)] (if zip-stream @@ -686,27 +524,5 @@ - - -; A testbench for using the main API function -(comment - (require '[polismath.runner :as runner]) - (def darwin (:darwin runner/system)) - runner/system - (export-conversation runner/system - {;;:zid 310273 - :zinvite "7scufp" - :format :csv - :filename "cljwebdev.zip" - :math-env "prod"}) - (export-conversation darwin - {;;:zid 310273 - :zinvite "7scufp" - :format :excel - :filename "cljwebdev.xlsx" - :math-env "prod"}) - :end-comment) - - :ok diff --git a/math/src/polismath/math/clusters.clj b/math/src/polismath/math/clusters.clj index 4b8fffb624..1b4059e6c1 100644 --- a/math/src/polismath/math/clusters.clj +++ b/math/src/polismath/math/clusters.clj @@ -2,20 +2,15 @@ (ns polismath.math.clusters (:refer-clojure :exclude [* - + == /]) - (:require [taoensso.timbre :as log] - [plumbing.core :as pc - :refer (fnk map-vals <-)] - [plumbing.graph :as gr] - [clojure.tools.trace :as tr] - ; [alex-and-georges.debug-repl :as dbr] - [polismath.utils :as utils] - [polismath.math.stats :as stats] - [polismath.math.named-matrix :as nm] - [clojure.spec.alpha :as s] - [clojure.core.matrix :as matrix] - [clojure.core.matrix.stats :as matrix-stats] - [clojure.core.matrix.selection :as matrix.selection] - [clojure.core.matrix.operators :refer :all])) + (:require + [clojure.core.matrix :as matrix] + [clojure.core.matrix.operators :refer [* - / max min]] + [clojure.core.matrix.stats :as matrix-stats] + [clojure.spec.alpha :as s] + [plumbing.core :as pc] + [polismath.math.named-matrix :as nm] + [polismath.utils :as utils] + [taoensso.timbre :as log])) ;; We may want to make formal that it's an int? Cause we add later? @@ -45,11 +40,11 @@ "Find the closest cluster and append item (mem_id, vector) to it" [clusts item] (let [[clst-id clst] (apply min-key - (fn [[clst-id clst]] - (matrix/distance (last item) (:center clst))) - clusts)] + (fn [[_clst-id clst]] + (matrix/distance (last item) (:center clst))) + clusts)] (assoc clusts clst-id - (clst-append clst item)))) + (clst-append clst item)))) (defn init-clusters @@ -61,7 +56,7 @@ vec ;; Have to cast to vec before we can distinct cause rows gives us a matrixlist distinct (map-indexed - (fn [id position] {:id id :members [] :center (matrix/matrix position)})) + (fn [id position] {:id id :members [] :center (matrix/matrix position)})) (take k))) @@ -71,9 +66,9 @@ [clsts1 clsts2 & {:keys [threshold] :or {threshold 0.01}}] (letfn [(cntrs [clsts] (sort (map :center clsts)))] (every? - (fn [[x y]] - (< (matrix/distance x y) threshold)) - (utils/zip (cntrs clsts1) (cntrs clsts2))))) + (fn [[x y]] + (< (matrix/distance x y) threshold)) + (utils/zip (cntrs clsts1) (cntrs clsts2))))) (defn cleared-clusters @@ -99,12 +94,12 @@ [mat & {:keys [weights]}] (if weights (weighted-mean - (* (/ (count weights) (pc/sum weights)) - (reduce - (fn [m [row-i weight]] - (matrix/multiply-row m row-i weight)) - mat - (utils/with-indices weights)))) + (* (/ (count weights) (pc/sum weights)) + (reduce + (fn [m [row-i weight]] + (matrix/multiply-row m row-i weight)) + mat + (utils/with-indices weights)))) (matrix-stats/mean (matrix/matrix mat)))) ; It's a vector... @@ -121,14 +116,9 @@ :weights (when weights (reduce - #(conj %1 (weights %2)) - [] - (nm/rownames nmat))))) - -;; Rename all above to weighted-mean* for profiling -;(defnp weighted-mean -; [& args] -; (apply weighted-mean* args)) + #(conj %1 (weights %2)) + [] + (nm/rownames nmat))))) (defn cluster-weights "Get a weights seq given a cluster with :members and a hash-map of weights. Returns nil @@ -143,29 +133,29 @@ "Performs one step of an iterative K-means: data-iter: seq of pairs (id, position), eg (pid, person-rating-row) clusters: array of clusters" - [data-iter k clusters & {:keys [weights]}] + [data-iter _k clusters & {:keys [weights]}] (->> data-iter ; Reduces a "blank" set of clusters w/ centers into clusters that have elements - (reduce add-to-closest (cleared-clusters clusters)) - vals + (reduce add-to-closest (cleared-clusters clusters)) + vals ; Filter out clusters that don't have any members (should maybe log on verbose?) - (filter #(> (count (:members %)) 0)) + (filter #(> (count (:members %)) 0)) ; Apply mean to get updated centers - (map (fn [clst] - (-> clst - (assoc :center (weighted-mean (:positions clst) - :weights (cluster-weights clst weights))) - (dissoc :positions)))))) + (map (fn [clst] + (-> clst + (assoc :center (weighted-mean (:positions clst) + :weights (cluster-weights clst weights))) + (dissoc :positions)))))) (defn recenter-clusters "Replace cluster centers with a center computed from new positions" [data clusters & {:keys [weights]}] (map - (fn [clst] - (assoc clst :center (weighted-mean (nm/rowname-subset data (:members clst)) - :weights weights))) - clusters)) + (fn [clst] + (assoc clst :center (weighted-mean (nm/rowname-subset data (:members clst)) + :weights weights))) + clusters)) (defn safe-recenter-clusters @@ -174,12 +164,12 @@ (as-> clusters clsts ; map every cluster to the newly centered cluster or to nil if there are no members in data (map - (fn [clst] - (let [rns (nm/safe-rowname-subset data (:members clst))] - (if (empty? (nm/rownames rns)) - nil - (assoc clst :center (weighted-mean rns :weights weights))))) - clsts) + (fn [clst] + (let [rns (nm/safe-rowname-subset data (:members clst))] + (if (empty? (nm/rownames rns)) + nil + (assoc clst :center (weighted-mean rns :weights weights))))) + clsts) ; Remove the nils, they break the math (remove nil? clsts) ; If nothing is left, make one great big cluster - so that things don't break in most-distal later @@ -205,26 +195,26 @@ (let [[dist clst-id id] ; find the maximum dist, clst-id, mem-id triple (apply max-key #(get % 0) - (map - (fn [mem] + (map + (fn [mem] ; Find the minimum distance, cluster-id pair, and add the member name to the end - (conj (apply min-key #(get % 0) - (map - #(vector (matrix/distance (nm/get-row-by-name data mem) (:center %)) (:id %)) - clusters)) - mem)) - (nm/rownames data)))] + (conj (apply min-key #(get % 0) + (map + #(vector (matrix/distance (nm/get-row-by-name data mem) (:center %)) (:id %)) + clusters)) + mem)) + (nm/rownames data)))] {:dist dist :clst-id clst-id :id id})) (defn uniqify-clusters [clusters] (reduce - (fn [clusters clst] - (let [identical-clst (first (filter #(= (:center clst) (:center %)) clusters))] - (if identical-clst - (assoc clusters (utils/typed-indexof clusters identical-clst) (merge-clusters identical-clst clst)) - (conj clusters clst)))) - [] clusters)) + (fn [clusters clst] + (let [identical-clst (first (filter #(= (:center clst) (:center %)) clusters))] + (if identical-clst + (assoc clusters (utils/typed-indexof clusters identical-clst) (merge-clusters identical-clst clst)) + (conj clusters clst)))) + [] clusters)) (defn clean-start-clusters @@ -256,17 +246,17 @@ (if (> (:dist outlier) 0) ; There is work to be done, so do it (recur - (-> + (-> ; first remove the most distal point from the cluster it was in; - (mapv - (fn [clst] - (assoc clst :members - (remove (set [(:id outlier)]) (:members clst)))) - clusters) + (mapv + (fn [clst] + (assoc clst :members + (remove (set [(:id outlier)]) (:members clst)))) + clusters) ; next add a new cluster containing only said point. - (conj {:id (inc (apply max (map :id clusters))) - :members [(:id outlier)] - :center (nm/get-row-by-name data (:id outlier))}))) + (conj {:id (inc (apply max (map :id clusters))) + :members [(:id outlier)] + :center (nm/get-row-by-name data (:id outlier))}))) ; Else just return recentered clusters clusters)) ; Else just return recentered clusters @@ -284,19 +274,6 @@ (set))) -(defn simplify-clsts - "Given a clustering, creates a set of member sets. This makes it easy to compare clusters for equality. - Optional `:trans` keyword args lets you perform a transformation to the member names included in member - sets." - [clsts & {:keys [trans] :or {trans identity}}] - {:members (map - (pc/fn->> :members (map trans) set) - clsts) - :center (map - (pc/fn->> :center (mapv #(utils/round-to % 4))) - clsts)}) - - ; Each cluster should have the shape {:id :members :center} (defn kmeans "Performs a k-means clustering." @@ -317,13 +294,13 @@ ([m] (dist-matrix m m)) ([m1 m2] (matrix/matrix - (map - (fn [r1] - (map - (fn [r2] - (matrix/distance r1 r2)) - m2)) - m1)))) + (map + (fn [r1] + (map + (fn [r2] + (matrix/distance r1 r2)) + m2)) + m1)))) (defn named-dist-matrix @@ -331,9 +308,9 @@ ([nm] (named-dist-matrix nm nm)) ([nm1 nm2] (nm/named-matrix - (nm/rownames nm1) - (nm/rownames nm2) - (dist-matrix (nm/get-matrix nm1) (nm/get-matrix nm2))))) + (nm/rownames nm1) + (nm/rownames nm2) + (dist-matrix (nm/get-matrix nm1) (nm/get-matrix nm2))))) (defn silhouette @@ -344,35 +321,35 @@ (let [dist-row (nm/rowname-subset distmat [member]) [a b] (reduce - (fn [[a b] clst] - (let [memb-clst? (some #{member} (:members clst)) - membs (remove #{member} (:members clst))] + (fn [[a b] clst] + (let [memb-clst? (some #{member} (:members clst)) + membs (remove #{member} (:members clst))] ; This is a little bit silly, but will basically trigger returning 0 if member is in a ; singleton cluster - (if (and memb-clst? (empty? membs)) - (reduced [1 1]) + (if (and memb-clst? (empty? membs)) + (reduced [1 1]) ; Otherwise, continue... - (as-> membs data + (as-> membs data ; Subset to just the columns for this clusters - (nm/colname-subset dist-row data) - (nm/get-matrix data) + (nm/colname-subset dist-row data) + (nm/get-matrix data) ; This is a 2D row vector; we want 1D, so take first - (first data) + (first data) ; Take the mean of the entries - (matrix-stats/mean data) - (if memb-clst? - [data b] - [a (min data (or b data))]))))) - [nil nil] - clusters)] + (matrix-stats/mean data) + (if memb-clst? + [data b] + [a (min data (or b data))]))))) + [nil nil] + clusters)] ; The actual silhouette computation (/ (- b a) (max b a)))) ([distmat clusters] (let [cluster-distmat (nm/rowname-subset distmat (mapcat :members clusters))] (weighted-mean - (map - (partial silhouette distmat clusters) - (nm/rownames cluster-distmat)))))) + (map + (partial silhouette distmat clusters) + (nm/rownames cluster-distmat)))))) (defn group-members @@ -389,7 +366,7 @@ (defn fold-clusters "Takes clusters -- a seq of maps `{:members :id :center}` -- and transforms into a single map `{:id :members :x :y :count}`, where each key points to a seq of the values associated with each - cluster. In particular, this is what's used to format base clusters for mongo uploading (for the + cluster. In particular, this is what's used to format base clusters for uploading (for the sake of compression)." [clusters] {:id (map :id clusters) @@ -402,39 +379,23 @@ (defn unfold-clusters "The inverse of `fold-clusters`; takes folded clusters and puts them into standard form. i.e. `(= identity (comp unfold-clusters fold-clusters))`" - [{:keys [members id x y] :as folded-clusters}] + [{:keys [members id x y]}] (map - (fn [ms id x y] - {:id id - :members ms - :center [x y]}) - members - id - x - y)) - - -(defn xy-clusters-to-nmat [clusters] - (let [nmat (nm/named-matrix)] - (nm/update-nmat - nmat - (apply concat ; flatten the list of lists below - (mapv - (fn [cluster] - (let [center (:center cluster) - id (:id cluster)] - ; Return some values that we can feed to update-nmat - [[id :x (first center)] - [id :y (second center)]])) - - clusters))))) + (fn [ms id x y] + {:id id + :members ms + :center [x y]}) + members + id + x + y)) (defn xy-clusters-to-nmat2 [clusters] (nm/named-matrix - (map :id clusters) ; row names - [:x :y] ; column names - (matrix/matrix (map :center clusters)))) + (map :id clusters) ; row names + [:x :y] ; column names + (matrix/matrix (map :center clusters)))) :ok diff --git a/math/src/polismath/math/conversation.clj b/math/src/polismath/math/conversation.clj index d2be01b4d6..031d399458 100644 --- a/math/src/polismath/math/conversation.clj +++ b/math/src/polismath/math/conversation.clj @@ -1,30 +1,22 @@ ;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . -(ns polismath.math.conversation +(ns ^{:clj-kondo/ignore [:unresolved-symbol]} + polismath.math.conversation (:refer-clojure :exclude [* - + == /]) (:require - [polismath.utils :as utils] - [polismath.math.pca :as pca] - [polismath.math.clusters :as clusters] - [polismath.math.repness :as repness] - [polismath.math.named-matrix :as nm] - [clojure.core.matrix :as matrix] - [clojure.spec.alpha :as s] - [clojure.tools.reader.edn :as edn] - [clojure.tools.trace :as tr] - [clojure.math.numeric-tower :as math] - [clojure.core.matrix :as matrix] - [clojure.core.matrix.operators :refer :all] - [plumbing.core :as plmb] - [plumbing.graph :as graph] - [bigml.sampling.simple :as sampling] - ;[alex-and-georges.debug-repl :as dbr] - [taoensso.timbre :as log] - [clojure.spec.gen.alpha :as gen] - [clojure.test.check.generators :as generators] - [tech.v3.dataset :as ds] - [tech.v3.datatype :as dt])) - ;[semantic-csv.core :as s.csv])) + [bigml.sampling.simple :as sampling] + [clojure.core.matrix :as matrix] + [clojure.core.matrix.operators :refer [* + - / max min]] + [clojure.spec.alpha :as s] + [plumbing.core :as plmb] + [plumbing.graph :as graph] + [polismath.math.clusters :as clusters] + [polismath.math.named-matrix :as nm] + [polismath.math.pca :as pca] + [polismath.math.repness :as repness] + [polismath.utils :as utils] + [taoensso.timbre :as log])) + ;; Starting to spec out our domain model here and build generators for the pieces @@ -36,24 +28,12 @@ (s/gen (s/int-in 0 gen-bound)))) (s/def ::zid (pos-int 20)) -(s/def ::tid (pos-int 100)) -(s/def ::pid (pos-int 100)) (s/def ::gid (pos-int 8)) -;; QUESTION How do we have a generator that gives us monotonically increasing values? -(s/def ::created (s/and int? pos?)) (s/def ::vote #{-1 0 1 -1.0 1.0 0.0}) - -(s/def ::vote-map - (s/keys :req-un [::zid ::tid ::pid ::vote ::created])) - -(s/def ::maybe-vote (s/or :missing nil? :voted ::vote)) - (s/def ::zid int?) (s/def ::last-vote-timestamp int?) (s/def ::group-votes (constantly true)) -(s/def ::sub-group-votes - (s/map-of ::gid ::group-votes)) (s/def ::group-clusters ::clusters/clustering) (s/def ::subgroup-clusters @@ -65,12 +45,7 @@ (s/map-of ::gid ::repness/clustering-repness)) (s/def ::rating-mat - ;(s/with-gen - (s/and ::nm/NamedMatrix)) - ;every element is a ::maybe-vote - ;#(every?)))) - ;; Let's just use a generator that generates votes and places them in a rating mat? - ;()) + (s/and ::nm/NamedMatrix)) (s/def ::raw-rating-mat ::rating-mat) @@ -85,37 +60,30 @@ :full ::full-conversation)) -(defn new-conv [] +(defn new-conv "Minimal structure upon which to perform conversation updates" + [] {:raw-rating-mat (nm/named-matrix) :last-vote-timestamp 0 :lastVoteTimestamp 0 :rating-mat (nm/named-matrix)}) -;; I think this is old and can be removed -;(defn choose-group-k [base-clusters] -; (let [len (count base-clusters)] -; (cond -; (< len 99) 3 -; :else 4))) - - (defn agg-bucket-votes-for-tid [bid-to-pid rating-mat filter-cond tid] (if-let [idx (nm/index (nm/get-col-index rating-mat) tid)] ; If we have data for the given comment... (let [pid-to-row (zipmap (nm/rownames rating-mat) (range (count (nm/rownames rating-mat)))) person-rows (nm/get-matrix rating-mat)] (mapv ; for each bucket - (fn [pids] - (->> pids + (fn [pids] + (->> pids ; get votes for the tid from each ptpt in group - (map (fn [pid] (get (get person-rows (pid-to-row pid)) idx))) + (map (fn [pid] (get (get person-rows (pid-to-row pid)) idx))) ; filter votes you don't want to count - (filter filter-cond) + (filter filter-cond) ; count - (count))) - bid-to-pid)) + (count))) + bid-to-pid)) ; Otherwise return an empty vector [])) @@ -135,105 +103,106 @@ (def base-conv-update-graph "Base of all conversation updates; handles default update opts and does named matrix updating" - {:opts' (plmb/fnk [opts] - "Merge in opts with the following defaults" - ;; TODO Answer and resolve this question: - ;; QUESTION Does it make senes to have the defaults here or in the config.edn or both duplicated? - (merge {:n-comps 2 ; does our code even generalize to others? - :pca-iters 100 - :base-iters 100 - :base-k 100 - :max-k 5 - :group-iters 100 - ;; These three in particular we should be able to tune quickly - :max-ptpts 100000 - :max-cmts 10000 - :group-k-buffer 4} + {:opts' + (plmb/fnk [opts] + "Merge in opts with the following defaults" + ;; TODO Answer and resolve this question: + ;; QUESTION Does it make senes to have the defaults here or in the config.edn or both duplicated? + (merge {:n-comps 2 ; does our code even generalize to others? + :pca-iters 100 + :base-iters 100 + :base-k 100 + :max-k 5 + :group-iters 100 + ;; These three in particular we should be able to tune quickly + :max-ptpts 100000 + :max-cmts 10000 + :group-k-buffer 4} opts)) :zid (plmb/fnk [conv votes] - (or (:zid conv) - (:zid (first votes)))) + (or (:zid conv) + (:zid (first votes)))) :last-vote-timestamp - (plmb/fnk [conv votes] - (apply max - (or (:last-vote-timestamp conv) 0) - (map :created votes))) + (plmb/fnk [conv votes] + (apply max + (or (:last-vote-timestamp conv) 0) + (map :created votes))) :customs (plmb/fnk [conv votes opts'] ; Decides whether there is room for new ptpts/cmts, and which votes should be allowed in ; based on which ptpts/cmts have already been seen. This is a simple prevention against ; conversations that get too large. Returns {:pids :tids :votes}, where the first two kv ; pairs are persisted and built upon and persisted; :votes is used downstream and tossed - (reduce - (fn [{:keys [pids tids] :as result} - {:keys [pid tid] :as vote}] - (let [pid-room (< (count pids) (:max-ptpts opts')) - tid-room (< (count tids) (:max-cmts opts')) - pid-in (pids pid) - tid-in (tids tid)] - (if (and (or pid-room pid-in) - (or tid-room tid-in)) - (assoc result - :pids (conj (:pids result) pid) - :tids (conj (:tids result) tid) - :votes (conj (:votes result) vote)) - result))) + (reduce + (fn [{:keys [pids tids] :as result} + {:keys [pid tid] :as vote}] + (let [pid-room (< (count pids) (:max-ptpts opts')) + tid-room (< (count tids) (:max-cmts opts')) + pid-in (pids pid) + tid-in (tids tid)] + (if (and (or pid-room pid-in) + (or tid-room tid-in)) + (assoc result + :pids (conj (:pids result) pid) + :tids (conj (:tids result) tid) + :votes (conj (:votes result) vote)) + result))) ; Customs collection off which to base reduction; note that votes get cleared out - (assoc (or (:customs conv) {:pids #{} :tids #{}}) - :votes []) - votes)) + (assoc (or (:customs conv) {:pids #{} :tids #{}}) + :votes []) + votes)) :keep-votes (plmb/fnk [customs] - (:votes customs)) + (:votes customs)) :raw-rating-mat (plmb/fnk [conv keep-votes] - (nm/update-nmat - (:raw-rating-mat conv) - (map (fn [v] (vector (:pid v) (:tid v) (:vote v))) keep-votes))) + (nm/update-nmat + (:raw-rating-mat conv) + (map (fn [v] (vector (:pid v) (:tid v) (:vote v))) keep-votes))) :rating-mat (plmb/fnk [conv raw-rating-mat] - ;; This if-let here is just a simple performance optimization - (let [mat - (if-let [mod-out (:mod-out conv)] - (nm/zero-out-columns raw-rating-mat mod-out) - raw-rating-mat)] - mat)) + ;; This if-let here is just a simple performance optimization + (let [mat + (if-let [mod-out (:mod-out conv)] + (nm/zero-out-columns raw-rating-mat mod-out) + raw-rating-mat)] + mat)) :tids (plmb/fnk [conv rating-mat] - (nm/colnames rating-mat)) + (nm/colnames rating-mat)) :n (plmb/fnk [rating-mat] - (count (nm/rownames rating-mat))) + (count (nm/rownames rating-mat))) :n-cmts (plmb/fnk [rating-mat] - (count (nm/colnames rating-mat))) + (count (nm/colnames rating-mat))) :user-vote-counts - (plmb/fnk [raw-rating-mat] + (plmb/fnk [raw-rating-mat] ; For deciding in-conv below; filter ptpts based on how much they've voted - (->> (mapv - (fn [rowname row] - [rowname (count (remove nil? row))]) - (nm/rownames raw-rating-mat) - (nm/get-matrix raw-rating-mat)) - (into {}))) + (->> (mapv + (fn [rowname row] + [rowname (count (remove nil? row))]) + (nm/rownames raw-rating-mat) + (nm/get-matrix raw-rating-mat)) + (into {}))) ;; Ugg... right, have to clarify that we don't want to drop this; are we leaving anything else out like this? :mod-out (plmb/fnk [conv] - (:mod-out conv)) + (:mod-out conv)) :mod-in (plmb/fnk [conv] - (:mod-in conv)) + (:mod-in conv)) :meta-tids (plmb/fnk [conv] - (:meta-tids conv)) + (:meta-tids conv)) ;; There should really be a nice way for us to specify that we want a full recompute on everything except in-conv, ;; since in meta-tids we don't want to loose people in that process. @@ -241,29 +210,29 @@ ; This keeps track of which ptpts are in the conversation (to be considered ; for base-clustering) based on home many votes they have. Once a ptpt is in, ; they will remain in. - (as-> (or (:in-conv conv) #{}) in-conv + (as-> (or (:in-conv conv) #{}) in-conv ; Start with whatever you have, and join it with anything that meets the criteria - (into in-conv - (map first - (filter - (fn [[rowname cnt]] + (into in-conv + (map first + (filter + (fn [[_rowname cnt]] ; We only start looking at a ptpt if they have rated either all the comments or at ; least 7 if there are more than 7 - (>= cnt (min 7 n-cmts))) - user-vote-counts))) + (>= cnt (min 7 n-cmts))) + user-vote-counts))) ; If you are left with fewer than 15 participants, take the top most contributing ; participants - (let [greedy-n 15 - n-in-conv (count in-conv)] - (if (< n-in-conv greedy-n) - (->> user-vote-counts - (remove - (fn [[k v]] (in-conv k))) - (sort-by (comp - second)) - (map first) - (take (- greedy-n n-in-conv)) - (into in-conv)) - in-conv))))}) + (let [greedy-n 15 + n-in-conv (count in-conv)] + (if (< n-in-conv greedy-n) + (->> user-vote-counts + (remove + (fn [[k _v]] (in-conv k))) + (sort-by (comp - second)) + (map first) + (take (- greedy-n n-in-conv)) + (into in-conv)) + in-conv))))}) ; End of base conv update @@ -271,9 +240,9 @@ (defn max-k-fn [data max-max-k] (min - max-max-k - (+ 2 - (int (/ (count (nm/rownames data)) 12))))) + max-max-k + (+ 2 + (int (/ (count (nm/rownames data)) 12))))) (defn group-votes "Returns a map of group-clusters ids to {:votes { {:A _ :D _ :S _}}} :n maps." @@ -281,28 +250,28 @@ (let [bid-to-index (zipmap (map :id base-clusters) (range))] (into {} - (map - (fn [{:keys [id members] :as group-cluster}] - (letfn [(count-fn [tid vote] - (->> - members - (mapv bid-to-index) - (mapv #(((votes-base tid) vote) %)) - (apply +)))] - [id - {:n-members (let [bids (set members)] + (map + (fn [{:keys [id members]}] + (letfn [(count-fn [tid vote] + (->> + members + (mapv bid-to-index) + (mapv #(((votes-base tid) vote) %)) + (apply +)))] + [id + {:n-members (let [bids (set members)] ; Add up the count of members in each base-cluster in this group-cluster - (->> base-clusters - (filter #(bids (:id %))) - (map #(count (:members %))) - (reduce + 0))) - :votes (plmb/map-from-keys - (fn [tid] - {:A (count-fn tid :A) - :D (count-fn tid :D) - :S (count-fn tid :S)}) - (keys votes-base))}])) - group-clusters)))) + (->> base-clusters + (filter #(bids (:id %))) + (map #(count (:members %))) + (reduce + 0))) + :votes (plmb/map-from-keys + (fn [tid] + {:A (count-fn tid :A) + :D (count-fn tid :D) + :S (count-fn tid :S)}) + (keys votes-base))}])) + group-clusters)))) (defn importance-metric @@ -319,12 +288,12 @@ [is-meta A P S E] ;; We square to deepen our bias (matrix/pow - (if is-meta - meta-priority - (* (importance-metric A P S E) - ;; scale by a factor which lets new comments bubble up - (+ 1 (* 8 (matrix/pow 2 (/ S -5)))))) - 2)) + (if is-meta + meta-priority + (* (importance-metric A P S E) + ;; scale by a factor which lets new comments bubble up + (+ 1 (* 8 (matrix/pow 2 (/ S -5)))))) + 2)) (comment @@ -341,9 +310,9 @@ (let [cmnt-proj (pca/pca-project-cmnts pca) cmnt-extremity (mapv - (fn [row] - (matrix/length row)) - (matrix/rows cmnt-proj))] + (fn [row] + (matrix/length row)) + (matrix/rows cmnt-proj))] (assoc pca :comment-projection (matrix/transpose cmnt-proj) :comment-extremity cmnt-extremity))) @@ -351,181 +320,181 @@ (def small-conv-update-graph "For computing small conversation updates (those without need for base clustering)" (merge - base-conv-update-graph - {:mat (plmb/fnk [rating-mat] + base-conv-update-graph + {:mat (plmb/fnk [rating-mat] ; swap nils for per column average - most things need the 0s, but repness needs the nils - (let [mat (nm/get-matrix rating-mat) - ;; TODO column-averages should be a separate build task - ;; Can toggle here for nil? or (nil or = 0) + (let [mat (nm/get-matrix rating-mat) + ;; TODO column-averages should be a separate build task + ;; Can toggle here for nil? or (nil or = 0) ;replace? #(or (nil? %) (= 0 %)) - replace? nil? - column-averages - (mapv - (fn [col] - (let [col (remove replace? col)] - (double - (/ (reduce + col) - (count col))))) - (matrix/columns mat))] - (mapv - (fn [row] - (mapv - (fn [i x] - (if (replace? x) (get column-averages i) x)) - (range) - row)) - mat))) - :pca (plmb/fnk [conv mat opts'] - (let [pca - (pca/wrapped-pca mat - (:n-comps opts') - :start-vectors (get-in conv [:pca :comps]) - :iters (:pca-iters opts'))] - (with-proj-and-extremtiy pca))) - - - :proj - (plmb/fnk [rating-mat pca] - (pca/sparsity-aware-project-ptpts (nm/get-matrix rating-mat) pca)) + replace? nil? + column-averages + (mapv + (fn [col] + (let [col (remove replace? col)] + (double + (/ (reduce + col) + (count col))))) + (matrix/columns mat))] + (mapv + (fn [row] + (mapv + (fn [i x] + (if (replace? x) (get column-averages i) x)) + (range) + row)) + mat))) + :pca (plmb/fnk [conv mat opts'] + (let [pca + (pca/wrapped-pca mat + (:n-comps opts') + :start-vectors (get-in conv [:pca :comps]) + :iters (:pca-iters opts'))] + (with-proj-and-extremtiy pca))) + + + :proj + (plmb/fnk [rating-mat pca] + (pca/sparsity-aware-project-ptpts (nm/get-matrix rating-mat) pca)) ;:cmnt-proj ;(plmb/fnk [pca] ; (pca/pca-project-cmnts pca)) - ;; QUESTION Just have proj return an nmat? - :proj-nmat - (plmb/fnk [rating-mat proj] - (nm/named-matrix (nm/rownames rating-mat) ["x" "y"] proj)) + ;; QUESTION Just have proj return an nmat? + :proj-nmat + (plmb/fnk [rating-mat proj] + (nm/named-matrix (nm/rownames rating-mat) ["x" "y"] proj)) - :base-clusters - (plmb/fnk [conv proj-nmat in-conv opts'] - (let [in-conv-mat (nm/rowname-subset proj-nmat in-conv)] - (sort-by :id - (clusters/kmeans in-conv-mat - (:base-k opts') - :last-clusters (:base-clusters conv) - :max-iters (:base-iters opts'))))) + :base-clusters + (plmb/fnk [conv proj-nmat in-conv opts'] + (let [in-conv-mat (nm/rowname-subset proj-nmat in-conv)] + (sort-by :id + (clusters/kmeans in-conv-mat + (:base-k opts') + :last-clusters (:base-clusters conv) + :max-iters (:base-iters opts'))))) - :base-clusters-proj - (plmb/fnk [base-clusters] - (clusters/xy-clusters-to-nmat2 base-clusters)) + :base-clusters-proj + (plmb/fnk [base-clusters] + (clusters/xy-clusters-to-nmat2 base-clusters)) - :bucket-dists - (plmb/fnk [base-clusters-proj] - (clusters/named-dist-matrix base-clusters-proj)) + :bucket-dists + (plmb/fnk [base-clusters-proj] + (clusters/named-dist-matrix base-clusters-proj)) - :base-clusters-weights - (plmb/fnk [base-clusters] - (into {} - (map - (fn [clst] - [(:id clst) (count (:members clst))]) - base-clusters))) + :base-clusters-weights + (plmb/fnk [base-clusters] + (into {} + (map + (fn [clst] + [(:id clst) (count (:members clst))]) + base-clusters))) - ;; Here we compute the top level clusters; These are the traditional clusters we've been using for some time now. - ;; Below we'll use these as the basis for a second level of subgroup clusters + ;; Here we compute the top level clusters; These are the traditional clusters we've been using for some time now. + ;; Below we'll use these as the basis for a second level of subgroup clusters ; Compute group-clusters for multiple k values - :group-clusterings - (plmb/fnk [conv base-clusters-weights base-clusters-proj opts'] - (plmb/map-from-keys - (fn [k] - (sort-by :id - (clusters/kmeans base-clusters-proj k - :last-clusters + :group-clusterings + (plmb/fnk [conv base-clusters-weights base-clusters-proj opts'] + (plmb/map-from-keys + (fn [k] + (sort-by :id + (clusters/kmeans base-clusters-proj k + :last-clusters ; A little pedantic here in case no clustering yet for this k - (when-let [last-clusterings (:group-clusterings conv)] - (last-clusterings k)) - :cluster-iters (:group-iters opts') - :weights base-clusters-weights))) - (range 2 (inc (max-k-fn base-clusters-proj (:max-k opts')))))) + (when-let [last-clusterings (:group-clusterings conv)] + (last-clusterings k)) + :cluster-iters (:group-iters opts') + :weights base-clusters-weights))) + (range 2 (inc (max-k-fn base-clusters-proj (:max-k opts')))))) ; Compute silhouette values for the various clusterings - :group-clusterings-silhouettes - (plmb/fnk [group-clusterings bucket-dists] - (plmb/map-vals (partial clusters/silhouette bucket-dists) group-clusterings)) + :group-clusterings-silhouettes + (plmb/fnk [group-clusterings bucket-dists] + (plmb/map-vals (partial clusters/silhouette bucket-dists) group-clusterings)) ; This smooths changes in cluster counts (K-vals) by remembering what the last K was, and only changing ; after (:group-k-buffer opts') many times on a new K value - :group-k-smoother - (plmb/fnk - [conv group-clusterings group-clusterings-silhouettes opts'] - (let [{:keys [last-k last-k-count smoothed-k] :or {last-k-count 0}} - (:group-k-smoother conv) - count-buffer (:group-k-buffer opts') + :group-k-smoother + (plmb/fnk + [conv group-clusterings group-clusterings-silhouettes opts'] + (let [{:keys [last-k last-k-count smoothed-k] :or {last-k-count 0}} + (:group-k-smoother conv) + count-buffer (:group-k-buffer opts') ; Find best K value for current data, given silhouette - this-k (apply max-key group-clusterings-silhouettes (keys group-clusterings)) + this-k (apply max-key group-clusterings-silhouettes (keys group-clusterings)) ; If this and last K values are the same, increment counter - same (if last-k (= this-k last-k) false) - this-k-count (if same (+ last-k-count 1) 1) + same (if last-k (= this-k last-k) false) + this-k-count (if same (+ last-k-count 1) 1) ; if seen > buffer many times, switch, OW, take last smoothed - smoothed-k (if (>= this-k-count count-buffer) - this-k - (if smoothed-k smoothed-k this-k))] - {:last-k this-k - :last-k-count this-k-count - :smoothed-k smoothed-k})) + smoothed-k (if (>= this-k-count count-buffer) + this-k + (if smoothed-k smoothed-k this-k))] + {:last-k this-k + :last-k-count this-k-count + :smoothed-k smoothed-k})) ; Pick the cluster corresponding to smoothed K value from group-k-smoother - :group-clusters - (plmb/fnk [group-clusterings group-k-smoother] - (get group-clusterings - (:smoothed-k group-k-smoother))) - - - ;; Now we're going to do the same thing for subclusters, or 2-level, 2-down hierarchical clustering - ;; This will more or less look the same, except that we'll have to do it for each group - - ;; Compute subgroup-clusters for each group id, for multiple k values, returning a nested map of: - ;; {:group-id {:k [clusters] ...} ...} - :subgroup-clusterings - (plmb/fnk [conv base-clusters-weights base-clusters-proj group-clusters opts'] - (into {} - ;; For each group cluster id - (map - (fn [group-cluster] - (let [gid (:id group-cluster) - group-members (:members group-cluster) - group-cluster-proj (nm/rowname-subset base-clusters-proj group-members)] - ;; For each k value... - ;; TODO Here we should really: - ;; * not return anyting if group cluster less than 10% (or some such) of population - ;; * not return anything if group cluster generally too small to subcluster (10?) - ;; * how do we handle these possibilties downstream? - [gid - (plmb/map-from-keys - (fn [k] - ;; We sort by id just to have a canonical repr - (sort-by - :id - (clusters/kmeans group-cluster-proj k - ;; This is where we grab the last-clusters from the last update's conv - :last-clusters (get-in conv [:subgroup-clusterings gid k]) ;; ok if nil - ;; TODO: Really need to properly account for what happens when group k changes... - ;; QUESTION: Add subgroup iters? For now assume same parameter value as group; level - :cluster-iters (:group-iters opts') - :weights base-clusters-weights))) - (range 2 (inc (max-k-fn group-cluster-proj (:max-k opts')))))])) - group-clusters))) + :group-clusters + (plmb/fnk [group-clusterings group-k-smoother] + (get group-clusterings + (:smoothed-k group-k-smoother))) + + + ;; Now we're going to do the same thing for subclusters, or 2-level, 2-down hierarchical clustering + ;; This will more or less look the same, except that we'll have to do it for each group + + ;; Compute subgroup-clusters for each group id, for multiple k values, returning a nested map of: + ;; {:group-id {:k [clusters] ...} ...} + :subgroup-clusterings + (plmb/fnk [conv base-clusters-weights base-clusters-proj group-clusters opts'] + (into {} + ;; For each group cluster id + (map + (fn [group-cluster] + (let [gid (:id group-cluster) + group-members (:members group-cluster) + group-cluster-proj (nm/rowname-subset base-clusters-proj group-members)] + ;; For each k value... + ;; TODO Here we should really: + ;; * not return anyting if group cluster less than 10% (or some such) of population + ;; * not return anything if group cluster generally too small to subcluster (10?) + ;; * how do we handle these possibilties downstream? + [gid + (plmb/map-from-keys + (fn [k] + ;; We sort by id just to have a canonical repr + (sort-by + :id + (clusters/kmeans group-cluster-proj k + ;; This is where we grab the last-clusters from the last update's conv + :last-clusters (get-in conv [:subgroup-clusterings gid k]) ;; ok if nil + ;; TODO: Really need to properly account for what happens when group k changes... + ;; QUESTION: Add subgroup iters? For now assume same parameter value as group; level + :cluster-iters (:group-iters opts') + :weights base-clusters-weights))) + (range 2 (inc (max-k-fn group-cluster-proj (:max-k opts')))))])) + group-clusters))) ; Compute silhouette values for the various clusterings - :subgroup-clusterings-silhouettes - (plmb/fnk [subgroup-clusterings bucket-dists] - (plmb/map-vals - (partial - plmb/map-vals - (partial clusters/silhouette bucket-dists)) - subgroup-clusterings)) - - ;; Should call this the k-selector: + :subgroup-clusterings-silhouettes + (plmb/fnk [subgroup-clusterings bucket-dists] + (plmb/map-vals + (partial + plmb/map-vals + (partial clusters/silhouette bucket-dists)) + subgroup-clusterings)) + + ;; Should call this the k-selector: ; This smooths changes in cluster counts (K-vals) by remembering what the last K was, and only changing ; after (:group-k-buffer opts') many times on a new K value - :subgroup-k-smoother - (plmb/fnk - [conv subgroup-clusterings subgroup-clusterings-silhouettes opts'] - (into {} - (map + :subgroup-k-smoother + (plmb/fnk + [conv subgroup-clusterings subgroup-clusterings-silhouettes opts'] + (into {} + (map (fn [[gid group-subgroup-clusterings]] (let [group-subgroup-silhouettes (get subgroup-clusterings-silhouettes gid) {:keys [last-k last-k-count smoothed-k] :or {last-k-count 0}} @@ -548,158 +517,159 @@ :smoothed-k smoothed-k}])) subgroup-clusterings))) - ;; This is a little different from the group version above; - ;; For each group, we take the subgroup clustering the best silhouette, and keep it. - ;; To each of the clusters in this clustering, we assoc the :parent-id of that cluster. - ;; We end up with a map that looks like { clusters}, where clusters looks as it does for `:group-clusters`, - ;; excepting the each cluster has a `:parent-id ` attr/value pair. - :subgroup-clusters - (plmb/fnk [subgroup-clusterings subgroup-k-smoother] - (into - {} - (map - (fn [[gid group-subgroup-clusterings]] - (if-let [smoothed-k (get-in subgroup-k-smoother [gid :smoothed-k])] - (do - (log/debug "Found smoothed-k:" smoothed-k) - [gid - (map - (plmb/fn-> (assoc :parent-id gid)) - (get group-subgroup-clusterings smoothed-k))]) - (log/warn "Didn't find smoothed-k for gid:" gid))) - subgroup-clusterings))) - - - - ;; a vector of member vectors, sorted by base cluster id - :bid-to-pid (plmb/fnk [base-clusters] - (mapv :members (sort-by :id base-clusters))) - - ;; returns {tid { - ;; :agree [0 4 2 0 6 0 0 1] - ;; :disagree [3 0 0 1 0 23 0 ]} - ;; where the indices in the arrays correspond NOT directly to the bid, but to the index of the - ;; corresponding bid in a hypothetically sorted list of the base cluster ids - :votes-base (plmb/fnk [bid-to-pid raw-rating-mat] - (->> raw-rating-mat - nm/colnames - (plmb/map-from-keys - (fn [tid] - {:A (agg-bucket-votes-for-tid bid-to-pid raw-rating-mat utils/agree? tid) - :D (agg-bucket-votes-for-tid bid-to-pid raw-rating-mat utils/disagree? tid) - :S (agg-bucket-votes-for-tid bid-to-pid raw-rating-mat number? tid)})))) - - - ;; In most or all of the below, we need to do things for both groups and subgroups. However, the logic for these - ;; isn't well refactored. There should ideally be some representation of the computations operating on group - ;; clusters that naturally generalizes or can be applied to the nesting of the subgroup-clusters. However, it also - ;; raises the question of how we could be structuring the subgroup clusters data differently to facilitate simpler - ;; provessing. There's some tradeoff between ease of initial processing (dealing with ids and such) and ease of - ;; downstream processing and data consumption. Will hvae to strike a balance here... For now, again, things are a - ;; little verbose, but you can hopefully see some of the patters emerging for where these things may generalize. + ;; This is a little different from the group version above; + ;; For each group, we take the subgroup clustering the best silhouette, and keep it. + ;; To each of the clusters in this clustering, we assoc the :parent-id of that cluster. + ;; We end up with a map that looks like { clusters}, where clusters looks as it does for `:group-clusters`, + ;; excepting the each cluster has a `:parent-id ` attr/value pair. + :subgroup-clusters + (plmb/fnk [subgroup-clusterings subgroup-k-smoother] + (into + {} + (map + (fn [[gid group-subgroup-clusterings]] + (if-let [smoothed-k (get-in subgroup-k-smoother [gid :smoothed-k])] + (do + (log/debug "Found smoothed-k:" smoothed-k) + [gid + (map + (plmb/fn-> (assoc :parent-id gid)) + (get group-subgroup-clusterings smoothed-k))]) + (log/warn "Didn't find smoothed-k for gid:" gid))) + subgroup-clusterings))) + + + + ;; a vector of member vectors, sorted by base cluster id + :bid-to-pid (plmb/fnk [base-clusters] + (mapv :members (sort-by :id base-clusters))) + + ;; returns {tid { + ;; :agree [0 4 2 0 6 0 0 1] + ;; :disagree [3 0 0 1 0 23 0 ]} + ;; where the indices in the arrays correspond NOT directly to the bid, but to the index of the + ;; corresponding bid in a hypothetically sorted list of the base cluster ids + :votes-base (plmb/fnk [bid-to-pid raw-rating-mat] + (->> raw-rating-mat + nm/colnames + (plmb/map-from-keys + (fn [tid] + {:A (agg-bucket-votes-for-tid bid-to-pid raw-rating-mat utils/agree? tid) + :D (agg-bucket-votes-for-tid bid-to-pid raw-rating-mat utils/disagree? tid) + :S (agg-bucket-votes-for-tid bid-to-pid raw-rating-mat number? tid)})))) + + + ;; In most or all of the below, we need to do things for both groups and subgroups. However, the logic for these + ;; isn't well refactored. There should ideally be some representation of the computations operating on group + ;; clusters that naturally generalizes or can be applied to the nesting of the subgroup-clusters. However, it also + ;; raises the question of how we could be structuring the subgroup clusters data differently to facilitate simpler + ;; provessing. There's some tradeoff between ease of initial processing (dealing with ids and such) and ease of + ;; downstream processing and data consumption. Will hvae to strike a balance here... For now, again, things are a + ;; little verbose, but you can hopefully see some of the patters emerging for where these things may generalize. ; {gid {:votes { {A _ D _ S}}}} - :group-votes - (plmb/fnk [group-clusters base-clusters votes-base] - (group-votes group-clusters base-clusters votes-base)) - ;; ? - :subgroup-votes - (plmb/fnk [subgroup-clusters base-clusters votes-base] - (->> subgroup-clusters - (plmb/map-vals - (fn [subgroup-clusters'] - (group-votes subgroup-clusters' base-clusters votes-base))))) + :group-votes + (plmb/fnk [group-clusters base-clusters votes-base] + (group-votes group-clusters base-clusters votes-base)) + ;; ? + :subgroup-votes + (plmb/fnk [subgroup-clusters base-clusters votes-base] + (->> subgroup-clusters + (plmb/map-vals + (fn [subgroup-clusters'] + (group-votes subgroup-clusters' base-clusters votes-base))))) ; {tid consensus} - :group-aware-consensus - (plmb/fnk [group-votes] - (let [tid-gid-probs - (reduce + :group-aware-consensus + (plmb/fnk [group-votes] + (let [tid-gid-probs + #_{:clj-kondo/ignore [:type-mismatch]} + (reduce (fn [result [gid gid-stats]] (reduce - (fn [result [tid {:keys [A S] :or {A 0 S 0}}]] - (let [prob (/ (+ A 1.0) (+ S 2.0))] - (assoc-in result [tid gid] prob))) - result - (:votes gid-stats))) - ;; +1 acts as a dumb prior + (fn [result [tid {:keys [A S] :or {A 0 S 0}}]] + (let [prob (/ (+ A 1.0) (+ S 2.0))] + (assoc-in result [tid gid] prob))) + result + (:votes gid-stats))) + ;; +1 acts as a dumb prior {} group-votes) - tid-consensus - (plmb/map-vals + tid-consensus + (plmb/map-vals (fn [tid-stats] (->> tid-stats (map second) (reduce *))) tid-gid-probs)] - tid-consensus)) - - :comment-priorities - (plmb/fnk [conv group-votes pca tids meta-tids] - (let [group-votes (:group-votes conv) - extremities (into {} (map vector tids (:comment-extremity pca)))] - (plmb/map-from-keys - (fn [tid] - (let [{:as total-votes :keys [A D S P]} - ;; reduce over votes per group, already aggregated - (reduce - (fn [votes [gid data]] - ;; not sure why we have to do the or here? how would this ever come up nil? small - ;; convs? - (let [{:as data :keys [A S D] :or {A 0 S 0 D 0}} (get-in data [:votes tid]) - data (assoc data :P (+ (- S (+ A D))))] - ;; Add in each of the data's kv count pairs - (reduce - (fn [votes' [k v]] - (update votes' k + v)) - votes - data))) - {:A 0 :D 0 :S 0 :P 0} - group-votes) - extremity (or (get extremities tid) - (do - (log/warn "No extremity for tid" tid "zid" (:zid conv)) - 0)) - ;; Use 0 as the default when meta-tids is null or doesn't contain the tid - meta-tid-value (if meta-tids - (get meta-tids tid 0) - 0)] - (priority-metric meta-tid-value A P S extremity))) - tids))) - - - ;; ATTENTION! The following uses of :mod-out should be ideally taking into account strict/vs non-strict - :repness - (plmb/fnk [conv rating-mat group-clusters base-clusters] - (-> (repness/conv-repness rating-mat group-clusters base-clusters) - (repness/select-rep-comments (:mod-out conv)))) - :subgroup-repness - (plmb/fnk [conv rating-mat subgroup-clusters base-clusters] - (->> subgroup-clusters - (plmb/map-vals - (fn [subgroup-clusters'] - (-> (repness/conv-repness rating-mat subgroup-clusters' base-clusters) - (repness/select-rep-comments (:mod-out conv))))))) - - :ptpt-stats - (plmb/fnk [group-clusters base-clusters proj-nmat user-vote-counts] - (repness/participant-stats group-clusters base-clusters proj-nmat user-vote-counts)) - :subgroup-ptpt-stats - (plmb/fnk [subgroup-clusters base-clusters proj-nmat user-vote-counts] - (->> subgroup-clusters - (plmb/map-vals - (fn [subgroup-clusters'] - (repness/participant-stats subgroup-clusters' base-clusters proj-nmat user-vote-counts))))) - - - :consensus - (plmb/fnk [conv rating-mat] - (-> (repness/consensus-stats rating-mat) - (repness/select-consensus-comments (:mod-out conv)))) + tid-consensus)) + + :comment-priorities + (plmb/fnk [conv group-votes pca tids meta-tids] + (let [group-votes (:group-votes conv) + extremities (into {} (map vector tids (:comment-extremity pca)))] + (plmb/map-from-keys + (fn [tid] + (let [{:keys [A P]} + ;; reduce over votes per group, already aggregated + (reduce + (fn [votes [_gid data]] + ;; not sure why we have to do the or here? how would this ever come up nil? small + ;; convs? + (let [{:as data :keys [A S D] :or {A 0 S 0 D 0}} (get-in data [:votes tid]) + data (assoc data :P (+ (- S (+ A D))))] + ;; Add in each of the data's kv count pairs + (reduce + (fn [votes' [k v]] + (update votes' k + v)) + votes + data))) + {:A 0 :D 0 :S 0 :P 0} + group-votes) + extremity (or (get extremities tid) + (do + (log/warn "No extremity for tid" tid "zid" (:zid conv)) + 0)) + ;; Use 0 as the default when meta-tids is null or doesn't contain the tid + meta-tid-value (if meta-tids + (get meta-tids tid 0) + 0)] + (priority-metric meta-tid-value A P S extremity))) + tids))) + + + ;; ATTENTION! The following uses of :mod-out should be ideally taking into account strict/vs non-strict + :repness + (plmb/fnk [conv rating-mat group-clusters base-clusters] + (-> (repness/conv-repness rating-mat group-clusters base-clusters) + (repness/select-rep-comments (:mod-out conv)))) + :subgroup-repness + (plmb/fnk [conv rating-mat subgroup-clusters base-clusters] + (->> subgroup-clusters + (plmb/map-vals + (fn [subgroup-clusters'] + (-> (repness/conv-repness rating-mat subgroup-clusters' base-clusters) + (repness/select-rep-comments (:mod-out conv))))))) + + :ptpt-stats + (plmb/fnk [group-clusters base-clusters proj-nmat user-vote-counts] + (repness/participant-stats group-clusters base-clusters proj-nmat user-vote-counts)) + :subgroup-ptpt-stats + (plmb/fnk [subgroup-clusters base-clusters proj-nmat user-vote-counts] + (->> subgroup-clusters + (plmb/map-vals + (fn [subgroup-clusters'] + (repness/participant-stats subgroup-clusters' base-clusters proj-nmat user-vote-counts))))) + + + :consensus + (plmb/fnk [conv rating-mat] + (-> (repness/consensus-stats rating-mat) + (repness/select-consensus-comments (:mod-out conv)))) ; End of large-update - #_:end})) + #_:end})) @@ -711,8 +681,8 @@ :or {n-comps 2 iters 10 learning-rate 0.01}}] (let [rating-subset (utils/filter-by-index mat indices) part-pca (pca/powerit-pca rating-subset n-comps - :start-vectors (:comps pca) - :iters iters) + :start-vectors (:comps pca) + :iters iters) forget-rate (- 1 learning-rate) learn (fn [old-val new-val] (let [old-val (matrix/join old-val (repeat (- (matrix/dimension-count new-val 0) @@ -733,8 +703,8 @@ start (- (* slope start-x) start-y)] (fn [size] (max - (long (min (+ start (* slope size)) stop-y)) - start-y)))) + (long (min (+ start (* slope size)) stop-y)) + start-y)))) ; For now... Will want this constructed with opts eventually XXX (def sample-size (sample-size-fn 100 1500 1500 150000)) @@ -742,17 +712,17 @@ (def large-conv-update-graph "Same as small-conv-update-graph, but uses mini-batch PCA" (merge small-conv-update-graph - {:pca (plmb/fnk [conv mat opts'] - (let [n-ptpts (matrix/dimension-count mat 0) - sample-size (sample-size n-ptpts)] - (loop [pca (:pca conv) iter (:pca-iters opts')] - (let [rand-indices (take sample-size (sampling/sample (range n-ptpts) :generator :twister)) - pca ((partial-pca mat pca rand-indices) pca)] - (if (= iter 0) - ;; Then done, but don't forget to merge in the comment extremtiy, etc - (with-proj-and-extremtiy pca) - ;; Recur - (recur pca (dec iter)))))))})) + {:pca (plmb/fnk [conv mat opts'] + (let [n-ptpts (matrix/dimension-count mat 0) + sample-size (sample-size n-ptpts)] + (loop [pca (:pca conv) iter (:pca-iters opts')] + (let [rand-indices (take sample-size (sampling/sample (range n-ptpts) :generator :twister)) + pca ((partial-pca mat pca rand-indices) pca)] + (if (= iter 0) + ;; Then done, but don't forget to merge in the comment extremtiy, etc + (with-proj-and-extremtiy pca) + ;; Recur + (recur pca (dec iter)))))))})) (def eager-profiled-compiler @@ -790,15 +760,15 @@ (log/info (str "Starting conv-update for zid " zid ": N=" n-ptpts ", C=" n-cmts ", V=" (count votes))) (-> ; dispatch to the appropriate function based on either participant or comment count - ((cond - (or (> n-ptpts ptpt-cutoff) - (> n-cmts cmt-cutoff)) large-conv-update - :else small-conv-update) - {:conv conv :votes votes :opts opts}) - ;; This seems hackish... XXX + ((cond + (or (> n-ptpts ptpt-cutoff) + (> n-cmts cmt-cutoff)) large-conv-update + :else small-conv-update) + {:conv conv :votes votes :opts opts}) + ;; This seems hackish... XXX ; Remove the :votes key from customs; not needed for persistence - (assoc-in [:customs :votes] []) - (dissoc :keep-votes))))))) + (assoc-in [:customs :votes] []) + (dissoc :keep-votes))))))) @@ -833,28 +803,28 @@ ;; processing things in order for mod in then out vs out then in (let [mod-out (reduce - (fn [mod-out {:keys [tid is_meta mod]}] - (if (or is_meta (= mod -1)) - (conj mod-out tid) - (disj mod-out tid))) - (set (:mod-out conv)) - mods) + (fn [mod-out {:keys [tid is_meta mod]}] + (if (or is_meta (= mod -1)) + (conj mod-out tid) + (disj mod-out tid))) + (set (:mod-out conv)) + mods) mod-in (reduce - (fn [mod-in {:keys [tid is_meta mod]}] - (if (or is_meta (= mod 1)) - (conj mod-in tid) - (disj mod-in tid))) - (set (:mod-in conv)) - mods) + (fn [mod-in {:keys [tid is_meta mod]}] + (if (or is_meta (= mod 1)) + (conj mod-in tid) + (disj mod-in tid))) + (set (:mod-in conv)) + mods) meta-tids (reduce - (fn [meta-tids {:keys [tid is_meta]}] - (if is_meta - (conj meta-tids tid) - (disj meta-tids tid))) - (set (:meta-tids conv)) - mods)] + (fn [meta-tids {:keys [tid is_meta]}] + (if is_meta + (conj meta-tids tid) + (disj meta-tids tid))) + (set (:meta-tids conv)) + mods)] (-> conv (assoc :mod-out mod-out :mod-in mod-in @@ -875,8 +845,8 @@ [o ^java.io.Writer w] (.write w "#mikera.matrixx.Matrix ") (ipv-print-method - (mapv #(into [] %) o) - w)) + (mapv #(into [] %) o) + w)) (defmethod print-method mikera.vectorz.Vector [o ^java.io.Writer w] @@ -889,38 +859,16 @@ (ipv-print-method o w)) -(defn misc-reader [type] - (fn [& args] - {:type type - :args args})) - -; a reader that uses these custom printing formats -(defn read-vectorz-edn [text] - (edn/read-string - {:readers {'mikera.vectorz.Vector matrix/matrix - 'mikera.arrayz.NDArray matrix/matrix - 'mikera.matrixx.Matrix matrix/matrix - 'polismath.named-matrix.NamedMatrix nm/named-matrix-reader - 'object (misc-reader :object) - 'error (misc-reader :error)}} - text)) - - - (defn conv-update-dump "Write out conversation state, votes, computational opts and error for debugging purposes." [conv votes & [opts error]] (spit (str "errorconv." (. System (nanoTime)) ".edn") - (prn-str - {:conv (into {} - (assoc-in conv [:pca :center] (matrix/matrix (into [] (:center (:pca conv)))))) - :votes votes - :opts opts - :error (str error)}))) - - -(defn load-conv-update [filename] - (read-vectorz-edn (slurp filename))) + (prn-str + {:conv (into {} + (assoc-in conv [:pca :center] (matrix/matrix (into [] (:center (:pca conv)))))) + :votes votes + :opts opts + :error (str error)}))) :ok diff --git a/math/src/polismath/math/corr.clj b/math/src/polismath/math/corr.clj deleted file mode 100644 index cf18053082..0000000000 --- a/math/src/polismath/math/corr.clj +++ /dev/null @@ -1,247 +0,0 @@ -;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . - -(ns polismath.math.corr - (:refer-clojure :exclude [* - + == /]) - (:require - [taoensso.timbre :as log] - ;[plumbing.core :as pc - ; :refer (fnk map-vals <-)] - ;[plumbing.graph :as gr] - ;[clojure.tools.trace :as tr] - ;[polismath.utils :as utils] - ;[polismath.math.stats :as stats] - [polismath.math.named-matrix :as nm] - [clojure.spec.alpha :as s] - [clojure.core.matrix :as matrix] - ;[clojure.core.matrix.stats :as matrix-stats] - [clojure.core.matrix.selection :as matrix.selection] - [clojure.core.matrix.operators :refer :all] - ;; - ;[incanter.charts :as charts] - ;[polismath.conv-man :as conv-man] - [clojure.set :as set] - [polismath.components.postgres :as postgres])) - ;; Can't use ic-stats anymore; It's turned to shit and can't be imported in clj/deps.edn - ;[incanter.stats :as ic-stats])) - - - -;; The following is a naive hclust implementation for clustering comments using the rating matrix. -;; This is so that we can rearrange a correlation matrix according to the hclust ordering for one of them slick heatmap things. -;; This is not a particularly efficient algorithm, but it gets the job done for now. - -(defn -hclust - "Implements the inner recursive agglomeration step for the hclust function." - ([clusters distances] - (if (= (count clusters) 1) - clusters - (let [[c1 c2] (->> (for [c1 clusters - c2 clusters - ;; Make sure we don't traverse [c c], or both [x y] and [y x] - :when (< (:id c1) (:id c2))] - [c1 c2]) - (apply min-key - (fn [[c1 c2]] - (let [key [(:id c1) (:id c2)]] - (if-let [dist (get @distances key)] - dist - (let [dist (matrix/distance (:center c1) (:center c2))] - (swap! distances assoc key dist) - dist)))))) - size (+ (:size c1) (:size c2)) - clusters (-> clusters - (->> (remove #{c1 c2})) - (conj {:id (inc (apply max (map :id clusters))) - :children [c1 c2] - :members (concat (:members c1) (:members c2)) - :distance (get @distances [(:id c1) (:id c2)]) - :center (/ (+ (* (:size c1) (:center c1)) - (* (:size c2) (:center c2))) - size) - :size size}))] - (-hclust clusters)))) - ([clusters] - (-hclust clusters (atom {})))) - - -(defn hclust - "Performs hclust on a named matrix" - [nmatrix] - (log/debug "hclust on matrix of shape" (matrix/shape (nm/get-matrix nmatrix))) - (-hclust - (mapv (fn [id row] - {:center row :id id :size 1 :members [id]}) - (nm/rownames nmatrix) - (matrix/rows (nm/get-matrix nmatrix))))) - - -(defn flatten-hclust - "Extracts out the tip/leaf node ordering from the hclust results. - (This is rendered somewhat irrelevant by the fact that the hclust algorithm now tracks this data using :members attr)" - [clusters] - (mapcat - (fn [cluster] - (if-let [children (:children cluster)] - (flatten-hclust children) - [(:id cluster)])) - clusters)) - - -(defn blockify-corr-matrix - "Rearrange the given correlation matrix ({:comments :matrix}) such that" - [corr-matrix clusters] - (let [comments (:comments corr-matrix) - matrix (:matrix corr-matrix) - members (:members (first clusters)) - member-indices (mapv #(.indexOf comments %) members)] - {:matrix (matrix.selection/sel matrix member-indices member-indices) - :comments members})) - - -;; Need to remove nulls to compute cov/corr - -;; Ideally we'd have something like a null senstive implementation of corr/cov? Restrict to overlapping dimensions? -;; What to do about 0 variance comments (all pass/null)? -;; These return NaN; treat as 0? - - -;; These things really should be in the named matrix namespace? -(defn cleaned-nmat [nmat] - (nm/named-matrix - (nm/rownames nmat) - (nm/colnames nmat) - (matrix/matrix - (mapv (fn [row] (map #(or % 0) row)) - (matrix/rows (nm/get-matrix nmat)))))) - -(defn transpose-nmat [nmat] - (nm/named-matrix - (nm/colnames nmat) - (nm/rownames nmat) - (matrix/transpose (nm/get-matrix nmat)))) - -;(defn cleaned-matrix [conv] -; (mapv (fn [row] (map #(or % 0) row)) -; (matrix/rows (nm/get-matrix (:rating-mat conv))))) - - -;; The actual correlation matrix stuff - - -;; See above -;; Can't use ic-stats anymore; It's turned to shit and can't be imported in clj/deps.edn -;(defn correlation-matrix - ;[nmat] - ;{:matrix (ic-stats/correlation (matrix/matrix (nm/get-matrix nmat))) - ;:comments (nm/colnames nmat)}) - - -;; Here's some stuff for spitting out the actual results. - -(require '[cheshire.core :as cheshire]) - -(defn prepare-hclust-for-export - [clusters] - (mapv - (fn [cluster] - (-> cluster - (update :center (partial into [])))) - clusters)) - - -(defn default-tids - ;; Gonna need to get all of this under group clusters vs subgroups - [{:as conv :keys [repness consensus group-clusters rating-mat pca]}] - (let [{:keys [extremity]} pca - {:keys [agree disagree]} consensus] - (set - (concat - (map :tid (concat agree disagree)) - (mapcat - (fn [[gid gid-repness]] - (map :tid gid-repness)) - repness))))) - - -;; See above: -;; Can't use ic-stats anymore; It's turned to shit and can't be imported in clj/deps.edn -;(defn compute-corr - ;([conv tids] - ;(let [matrix (:rating-mat conv) - ;subset-matrix (if tids (nm/colname-subset matrix tids) matrix) - ;cleaned-matrix (cleaned-nmat subset-matrix) - ;transposed-matrix (transpose-nmat cleaned-matrix) - ;; note: taoensso.timbre.profiling was removed from timbre; use com.taoensso/tufte now - ;corr-mat (prof/profile :info :corr-mat (correlation-matrix cleaned-matrix)) - ;hclusters (prof/profile :info ::hclust (hclust transposed-matrix)) - ;corr-mat' (blockify-corr-matrix corr-mat hclusters) - ;corr-mat' - ;;; Prep for export... - ;(update corr-mat' :matrix (comp (partial mapv (fn [row] (into [] row))) - ;matrix/rows))] - ;corr-mat')) - ;([conv] - ;(compute-corr conv (default-tids conv)))) - - -(defn spit-json - [filename data] - (spit - filename - (cheshire/encode data))) - -(defn spit-hclust - [filename clusters] - (spit-json - filename - (prepare-hclust-for-export clusters))) - -;; We should add a little clarity on where things are regular matrices vs our {:comments :matrix} matrices for final export... -;; And probably just use named matrices everywhere -(defn spit-matrix - [filename matrix] - (spit-json - filename - (update matrix :matrix (comp (partial mapv (fn [row] (into [] row))) - matrix/rows)))) - - -;; We have metadata/tags on the argument symbols of function vars -;; So what if we built a macro that copied over entire namespaces of functions to a new ns, but binds them to some sort of default system? -;; System bind! -;; Then you can access a mirror api that has sans-1 arity versions of all the functions, so you don't have to think about system! -;; This could solve the battle between component and mount - - - - -(comment - ;; Here we're putting everything together - ;; This may not all be 100% correct, as it was copied over from the repl... but I ran through all but the spit and sanity checks pass - (require '[incanter.charts :as charts] - '[polismath.runner :as runner] - '[polismath.conv-man :as conv-man] - '[polismath.system :as system]) - ;(runner/run! system/base-system {:math-env :preprod}) - ;; Load the data for 15117 (zinvite 2ez5beswtc) - ;(def focus-id 15117) - (def focus-zinvite "36jajfnhhn") - (def focus-id 15228) - (def conv (conv-man/load-or-init (:conversation-manager runner/system) focus-id)) - - (compute-corr conv) - :ok - - ;conv - (matrix/shape (nm/get-matrix (:rating-mat conv))) - ;; Compute hclust on a cleaned, transposed rating matrix - (def tids (nm/rownames (:rating-mat conv))) - tids - (def corr-mat' (compute-corr (:darwin runner/system))) - ;; Spit this out - (spit-matrix (str focus-zinvite ".corrmat.json") corr-mat) - (spit-matrix (str focus-zinvite ".corrmat-whclust.json") corr-mat') - (spit-hclust (str focus-zinvite ".hclust.json") hclusters) - ;; All done - :endcomment) - diff --git a/math/src/polismath/math/named_matrix.clj b/math/src/polismath/math/named_matrix.clj index d19ace2cf5..785b86a004 100644 --- a/math/src/polismath/math/named_matrix.clj +++ b/math/src/polismath/math/named_matrix.clj @@ -1,12 +1,11 @@ ;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . (ns polismath.math.named-matrix - (:require [clojure.core.matrix :as matrix] - [clojure.spec.alpha :as s] - [clojure.spec.gen.alpha :as gen] - [clojure.test.check.generators :as generators]) - ;; Again, move to - (:use polismath.utils)) + (:require + [clojure.core.matrix :as matrix] + [clojure.spec.alpha :as s] + [clojure.spec.gen.alpha :as gen] + [polismath.utils :refer [filter-by-index]])) (defprotocol AutoIncIndex @@ -21,29 +20,18 @@ (deftype IndexHash - [^java.util.Vector names ^clojure.lang.PersistentArrayMap index-hash] + [^java.util.Vector names ^clojure.lang.PersistentArrayMap index-hash] AutoIncIndex - (get-names [this] (.names this)) - (next-index [this] (count (.names this))) - (index [this keyname] (index-hash keyname)) + (get-names [_this] (.names _this)) + (next-index [_this] (count (.names _this))) + (index [_this keyname] (index-hash keyname)) (append [this keyname] (if ((.index-hash this) keyname) this (IndexHash. - (conj (get-names this) keyname) - (assoc (.index-hash this) keyname (next-index this))))) + (conj (get-names this) keyname) + (assoc (.index-hash this) keyname (next-index this))))) (append-many [this keynames] - ; potentially faster - ;(let [uniq-kns (distinct keynames) - ;new-kns (remove (.index-hash this) uniq-kns)] - ;(IndexHash. - ;(into (.names this) new-kns) - ;(into (.index-hash) - ;(map - ;#(vector %1 (+ (count (.names this)) %2)) - ;new-kns - ;(range))))) - ; much simpler (reduce append this keynames)) (subset [this keynames] (let [kn-set (set keynames) @@ -55,7 +43,7 @@ [keynames] (let [uniq-kns (into [] (distinct keynames))] (IndexHash. uniq-kns - (into {} (map vector uniq-kns (range)))))) + (into {} (map vector uniq-kns (range)))))) (defprotocol PNamedMatrix @@ -85,21 +73,21 @@ (deftype NamedMatrix - [row-index col-index matrix] + [row-index col-index matrix] PNamedMatrix (update-nmat [this values] ; First find the row and column names that aren't yet in the data (let [[missing-rows missing-cols] (reduce - (fn [[missing-rows missing-cols] [row col value]] - [(if (nil? (index (.row-index this) row)) - (conj missing-rows row) - missing-rows) - (if (nil? (index (.col-index this) col)) - (conj missing-cols col) - missing-cols)]) - [[] []] - values) + (fn [[missing-rows missing-cols] [row col _value]] + [(if (nil? (index (.row-index this) row)) + (conj missing-rows row) + missing-rows) + (if (nil? (index (.col-index this) col)) + (conj missing-cols col) + missing-cols)]) + [[] []] + values) ; Construct new rowname and colname hash-indices new-row-index (append-many (.row-index this) missing-rows) new-col-index (append-many (.col-index this) missing-cols) @@ -107,26 +95,26 @@ new-col-count (count (set missing-cols))] ; Construct a new NamedMatrix (NamedMatrix. - new-row-index - new-col-index + new-row-index + new-col-index ; Construct new matrix - (as-> (.matrix this) mat - (if (= 0 (matrix/dimension-count mat 1)) + (as-> (.matrix this) mat + (if (= 0 (matrix/dimension-count mat 1)) ; If the matrix is empty, just create the shape needed - (matrix/coerce [[]] - (matrix/broadcast nil [new-row-count new-col-count])) + (matrix/coerce [[]] + (matrix/broadcast nil [new-row-count new-col-count])) ; OW, add padding of nils for new rows/cols - (-> mat - (add-padding 1 (count (set missing-cols))) - (add-padding 0 (count (set missing-rows))))) + (-> mat + (add-padding 1 (count (set missing-cols))) + (add-padding 0 (count (set missing-rows))))) ; Next assoc-in all of the new votes - (reduce - (fn [mat' [row col value]] - (let [row-i (index new-row-index row) - col-i (index new-col-index col)] - (assoc-in mat' [row-i col-i] value))) - mat - values))))) + (reduce + (fn [mat' [row col value]] + (let [row-i (index new-row-index row) + col-i (index new-col-index col)] + (assoc-in mat' [row-i col-i] value))) + mat + values))))) (rownames [this] (get-names (.row-index this))) (colnames [this] (get-names (.col-index this))) (get-matrix [this] (.matrix this)) @@ -136,40 +124,40 @@ (let [row-indices (map (partial index (.row-index this)) names) row-index (subset (.row-index this) names)] (NamedMatrix. - row-index - (.col-index this) - (filter-by-index (.matrix this) row-indices)))) + row-index + (.col-index this) + (filter-by-index (.matrix this) row-indices)))) (colname-subset [this names] (let [col-indices (map (partial index (.col-index this)) names) col-index (subset (.col-index this) names)] (NamedMatrix. - (.row-index this) - col-index - (matrix/select (.matrix this) :all col-indices))))) + (.row-index this) + col-index + (matrix/select (.matrix this) :all col-indices))))) (defn named-matrix "Generator function for a new named matrix" [& [rows cols matrix]] (NamedMatrix. - (index-hash (or rows [])) - (index-hash (or cols [])) - (or matrix [[]]))) + (index-hash (or rows [])) + (index-hash (or cols [])) + (or matrix [[]]))) (defn named-matrix-gen-of [gen] (gen/bind - (gen/tuple - (gen/such-that not-empty (gen/vector-distinct (gen/such-that pos? (gen/int)))) - (gen/such-that not-empty (gen/vector-distinct (gen/such-that pos? (gen/int))))) - (fn [[rownames colnames]] - (gen/fmap - (fn [matrix] (named-matrix rownames colnames matrix)) - (gen/vector - (gen/vector gen (count colnames)) - (count rownames)))))) + (gen/tuple + (gen/such-that not-empty (gen/vector-distinct (gen/such-that pos? (gen/int)))) + (gen/such-that not-empty (gen/vector-distinct (gen/such-that pos? (gen/int))))) + (fn [[rownames colnames]] + (gen/fmap + (fn [matrix] (named-matrix rownames colnames matrix)) + (gen/vector + (gen/vector gen (count colnames)) + (count rownames)))))) ;(gen/sample (named-matrix-gen-of (s/gen #{-1 0 1 nil}))) @@ -177,35 +165,23 @@ (s/def ::NamedMatrix (s/with-gen (s/and - (partial satisfies? PNamedMatrix) + (partial satisfies? PNamedMatrix) ;(comp get-matrix matrix/matrix?) - (comp matrix/vec? rownames) - (comp matrix/vec? colnames) - (comp distinct rownames) - (comp distinct colnames)) - ;; Silly... need way smarter generators here; And maybe actually the generator shouldn't be on this entity, since a named - ;(clojure.test.check.generators/elements) + (comp matrix/vec? rownames) + (comp matrix/vec? colnames) + (comp distinct rownames) + (comp distinct colnames)) #(named-matrix-gen-of (gen/double)))) -;(gen/sample (s/gen ::NamedMatrix)) - - -;(gen/frequency) -;(gen/double*) -;(gen/vector-of) -;(gen/sample (gen/double* {:min 0 :max 1 :NaN? false :infinite? false}) 100) - (defmethod print-method NamedMatrix [nm ^java.io.Writer w] (.write w - (str "#polismath.named-matrix.NamedMatrix " - "{:rownames " (into [] (rownames nm)) - " :colnames " (into [] (colnames nm)) - " :matrix " (get-matrix nm) - "}"))) - -;(gen/sample (s/gen (s/cat :nm ::NamedMatrix :columns coll?))) + (str "#polismath.named-matrix.NamedMatrix " + "{:rownames " (into [] (rownames nm)) + " :colnames " (into [] (colnames nm)) + " :matrix " (get-matrix nm) + "}"))) (s/fdef zero-out-columns :args (s/cat :nm ::NamedMatrix :columns coll?) @@ -220,13 +196,13 @@ col-index (get-col-index nm) m' (reduce - (fn [m' col] - ;; It's possible to is_meta out a comment that hasn't yet been repsonded to - (if-let [i (index col-index col)] - (matrix/set-column m' i 0) - m')) - m - columns)] + (fn [m' col] + ;; It's possible to is_meta out a comment that hasn't yet been repsonded to + (if-let [i (index col-index col)] + (matrix/set-column m' i 0) + m')) + m + columns)] (named-matrix rows cols m'))) ; Just trying to figure out how the hell function checking really works... @@ -235,31 +211,23 @@ (s/fdef whatever - :args (s/cat :x (s/and number? #(not= ##NaN %) #(not= 0 %))) + :args (s/cat :x (s/and number? #(not= ##NaN %) #(not (zero? %)))) :ret number? - :fn #(= (-> % :ret (* 2) (-> % :args :x)))) + :fn #(= (-> % :ret (* 2)) (-> % :args :x))) (defn whatever [x] (/ 2 x)) -;(gen/sample (s/gen number?)) (stest/check `whatever) - - -(defn named-matrix-reader - [{:keys [rownames colnames matrix]}] - (named-matrix rownames colnames matrix)) - - ; Put in interface? ... (defn safe-rowname-subset "This version of rowname-subset filters out negative indices, so that if not all names in row-names are in nmat, it just subsets to the rownames that are. Should scrap other one?" [nmat names] - (let [safe-names (filter + (let [_safe-names (filter (partial index (get-row-index nmat)) names)] (rowname-subset nmat names))) @@ -269,10 +237,11 @@ (matrix/get-row (get-matrix nmat) (index (get-row-index nmat) row-name))) -(defn inv-rowname-subset [nmat row-names] +(defn inv-rowname-subset "Returns named matrix which has been subset to all the rows not in row-names" + [nmat row-names] (rowname-subset nmat - (remove (set row-names) (rownames nmat)))) + (remove (set row-names) (rownames nmat)))) :ok diff --git a/math/src/polismath/math/pca.clj b/math/src/polismath/math/pca.clj index bc98d518fc..c4f2f33c1e 100644 --- a/math/src/polismath/math/pca.clj +++ b/math/src/polismath/math/pca.clj @@ -2,12 +2,12 @@ (ns polismath.math.pca (:refer-clojure :exclude [* - + == / min max]) - (:require [taoensso.timbre :as log] - [polismath.utils :as utils] - [clojure.core.match :refer [match]] - [clojure.core.matrix :as matrix] - [clojure.core.matrix.stats :as matrix-stats] - [clojure.core.matrix.operators :refer :all])) + (:require + [clojure.core.match :refer [match]] + [clojure.core.matrix :as matrix] + [clojure.core.matrix.operators :refer [* + - / max min]] + [clojure.core.matrix.stats :as matrix-stats] + [polismath.utils :as utils])) (matrix/set-current-implementation :vectorz) @@ -45,8 +45,8 @@ start-vector (or start-vector (repeatv n-cols 1)) ; XXX - this add extra cols to the start vector if we have new comments... should test start-vector (matrix/matrix - (concat start-vector - (repeatv (- n-cols (matrix/dimension-count start-vector 0)) 1)))] + (concat start-vector + (repeatv (- n-cols (matrix/dimension-count start-vector 0)) 1)))] (loop [iters iters start-vector start-vector last-eigval 0] (let [product-vector (xtxr data start-vector) eigval (matrix/length product-vector) @@ -79,7 +79,7 @@ (defn rand-starting-vec [data] ;; Should really throw a parallelizable random number generator in the equation here... ;; With seeds fed in and persisted... XXX - (matrix/matrix (for [x (range (matrix/dimension-count data 1))] (rand)))) + (matrix/matrix (for [_x (range (matrix/dimension-count data 1))] (rand)))) ; Will eventually also want to add last-pcs @@ -93,42 +93,35 @@ data-dim (min (matrix/row-count cntrd-data) (matrix/column-count cntrd-data))] {:center center :comps - (loop [data' cntrd-data n-comps' (min n-comps data-dim) pcs [] start-vectors start-vectors] + (loop [data' cntrd-data n-comps' (min n-comps data-dim) pcs [] start-vectors start-vectors] ; may eventually want to return eigenvals... - (let [start-vector (or (first start-vectors) (rand-starting-vec data)) - pc (power-iteration data' iters start-vector) - pcs (conj pcs pc)] - (if (= n-comps' 1) - pcs ; return if done - (let [data' (factor-matrix data' pc) - n-comps' (dec n-comps')] - (recur data' n-comps' pcs (rest start-vectors))))))})) + (let [start-vector (or (first start-vectors) (rand-starting-vec data)) + pc (power-iteration data' iters start-vector) + pcs (conj pcs pc)] + (if (= n-comps' 1) + pcs ; return if done + (let [data' (factor-matrix data' pc) + n-comps' (dec n-comps')] + (recur data' n-comps' pcs (rest start-vectors))))))})) (defn wrapped-pca "This function gracefully handles weird edge cases inherent in the messiness of real world data" - [data n-comps & {:keys [iters start-vectors] :as kwargs}] + [data n-comps & {:keys [start-vectors] :as kwargs}] (match (map (partial matrix/dimension-count data) [0 1]) [1 n-cols] {:center (matrix/matrix (repeatv n-comps 0)) :comps (into [(matrix/normalise (matrix/get-row data 0))] - (repeat (dec n-comps) (repeatv n-cols 0)))} - [n-rows 1] + (repeat (dec n-comps) (repeatv n-cols 0)))} + [_n-rows 1] {:center (matrix/matrix [0]) :comps (matrix/matrix [1])} :else - (utils/apply-kwargs powerit-pca data n-comps - (assoc kwargs :start-vectors - (if start-vectors - (map #(if (every? #{0 0.0} %) nil %) start-vectors) - nil))))) - - -(defn pca-project - "Apply the principal component projection specified by pcs to the data" - [data {:keys [comps center]}] - ; Here we map each row of data to its projection - (matrix/mmul (- data center) (matrix/transpose comps))) + (utils/apply-kwargs powerit-pca data n-comps + (assoc kwargs :start-vectors + (if start-vectors + (map #(if (every? #{0 0.0} %) nil %) start-vectors) + nil))))) (defn sparsity-aware-project-ptpt @@ -139,19 +132,19 @@ [n-votes p1 p2] ; (p1, p2) is the projection we build (reduce ; _-n is the nth entry in _ - (fn [[n-votes p1 p2] [x-n cntr-n pc1-n pc2-n]] + (fn [[n-votes p1 p2] [x-n cntr-n pc1-n pc2-n]] ; if we have voted, do the thing - (if x-n + (if x-n ; first subtract center - (let [x-n' (- x-n cntr-n)] + (let [x-n' (- x-n cntr-n)] ; then do a step in the dot product, and inc n-votes seen - [(inc n-votes) - (+ p1 (* x-n' pc1-n)) - (+ p2 (* x-n' pc2-n))]) + [(inc n-votes) + (+ p1 (* x-n' pc1-n)) + (+ p2 (* x-n' pc2-n))]) ; ... ow (if haven't voted) return what was there - [n-votes p1 p2])) - [0 0.0 0.0] - (utils/zip votes center pc1 pc2))] + [n-votes p1 p2])) + [0 0.0 0.0] + (utils/zip votes center pc1 pc2))] ; Now scale the projection by the following value, which pushes us out from the center (* (Math/sqrt (/ n-cmnts (max n-votes 1))) [p1 p2]))) @@ -165,17 +158,17 @@ (defn pca-project-cmnts - [{:as pca :keys [comps center]}] + [{:as pca :keys [comps]}] (let [n-cols (matrix/column-count comps)] (sparsity-aware-project-ptpts - (map - (fn [i] - (assoc - (vec (repeat n-cols nil)) - i - -1)) - (range n-cols)) - pca))) + (map + (fn [i] + (assoc + (vec (repeat n-cols nil)) + i + -1)) + (range n-cols)) + pca))) :ok diff --git a/math/src/polismath/math/repness.clj b/math/src/polismath/math/repness.clj index c0ff79360c..8ecd7752ec 100644 --- a/math/src/polismath/math/repness.clj +++ b/math/src/polismath/math/repness.clj @@ -1,23 +1,20 @@ ;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . -(ns polismath.math.repness - (:require [polismath.utils :as utils] - [polismath.math.stats :as stats] - [polismath.math.named-matrix :as nm] - [polismath.math.clusters :as clusters] - [clojure.spec.alpha :as s] - [clojure.core.matrix :as mat] - [clojure.core.matrix.operators :refer :all] - [clojure.tools.trace :as tr] - [plumbing.core :as pc :refer [fnk map-vals <-]] - [plumbing.graph :as gr] - [clojure.core.matrix :as matrix] - [taoensso.timbre :as log]) - ;[alex-and-georges.debug-repl :as dbr] - - (:refer-clojure :exclude [* - + == /])) - -(mat/set-current-implementation :vectorz) +(ns ^{:clj-kondo/ignore [:unresolved-symbol]} + polismath.math.repness + (:refer-clojure :exclude [* - + == /]) + (:require + [clojure.core.matrix :as matrix] + [clojure.core.matrix.operators :refer [* + - / max]] + [clojure.spec.alpha :as s] + [plumbing.core :as pc :refer [fnk map-vals]] + [plumbing.graph :as gr] + [polismath.math.clusters :as clusters] + [polismath.math.named-matrix :as nm] + [polismath.math.stats :as stats] + [polismath.utils :as utils])) + +(matrix/set-current-implementation :vectorz) (s/def ::repful-for @@ -29,9 +26,6 @@ (partial >= 1) (partial <= 0))) -(s/def ::test-val - (s/and number?)) - (s/def ::tid int?) (s/def ::gid int?) @@ -41,22 +35,15 @@ (partial s/valid? ::probability)) (s/def ::p-test number?) -(s/def ::repress - (s/and number? - (partial < 0))) - -(s/def ::comment-repness - (s/keys :req-un [::tid ::test-val ::repful-for ::n-success ::n-trials ::p-success ::p-test ::repness])) - (s/def ::clustering-repness (s/map-of ::gid (s/* ::repness))) (defn- count-votes - [votes & [vote]] "Utility function for counting the number of votes matching `vote`. Not specifying `vote` returns length of vote vector, sans any missed votes." + [votes & [vote]] (let [filt-fn (if vote #(= vote %) identity)] (count (filter filt-fn votes)))) @@ -65,14 +52,14 @@ ; The graph implementation is in a closure so that we don't reify classes every time function is called. ; This can cause PermGen space crashes. This is used for both consensus and repness comments. (gr/eager-compile - {:na (fnk [votes] (count-votes votes -1)) - :nd (fnk [votes] (count-votes votes 1)) - :ns (fnk [votes] (count-votes votes)) + {:na (fnk [votes] (count-votes votes -1)) + :nd (fnk [votes] (count-votes votes 1)) + :ns (fnk [votes] (count-votes votes)) ; XXX - Change when we flip votes!!! - :pa (fnk [na ns] (/ (+ 1 na) (+ 2 ns))) - :pd (fnk [nd ns] (/ (+ 1 nd) (+ 2 ns))) - :pat (fnk [na ns] (stats/prop-test na ns)) - :pdt (fnk [nd ns] (stats/prop-test nd ns))})) + :pa (fnk [na ns] (/ (+ 1 na) (+ 2 ns))) + :pd (fnk [nd ns] (/ (+ 1 nd) (+ 2 ns))) + :pat (fnk [na ns] (stats/prop-test na ns)) + :pdt (fnk [nd ns] (stats/prop-test nd ns))})) (defn- comment-stats @@ -88,16 +75,16 @@ See `(doc conv-repness)` for key details." [in-stats rest-stats] (assoc in-stats - :ra (/ (:pa in-stats) - (/ (+ 1 (pc/sum :na rest-stats)) - (+ 2 (pc/sum :ns rest-stats)))) - :rd (/ (:pd in-stats) - (/ (+ 1 (pc/sum :nd rest-stats)) - (+ 2 (pc/sum :ns rest-stats)))) - :rat (stats/two-prop-test (:na in-stats) (pc/sum :na rest-stats) - (:ns in-stats) (pc/sum :ns rest-stats)) - :rdt (stats/two-prop-test (:nd in-stats) (pc/sum :nd rest-stats) - (:ns in-stats) (pc/sum :ns rest-stats)))) + :ra (/ (:pa in-stats) + (/ (+ 1 (pc/sum :na rest-stats)) + (+ 2 (pc/sum :ns rest-stats)))) + :rd (/ (:pd in-stats) + (/ (+ 1 (pc/sum :nd rest-stats)) + (+ 2 (pc/sum :ns rest-stats)))) + :rat (stats/two-prop-test (:na in-stats) (pc/sum :na rest-stats) + (:ns in-stats) (pc/sum :ns rest-stats)) + :rdt (stats/two-prop-test (:nd in-stats) (pc/sum :nd rest-stats) + (:ns in-stats) (pc/sum :ns rest-stats)))) (defn conv-repness @@ -112,26 +99,26 @@ {:ids (map :id group-clusters) :tids (nm/colnames data) :stats - (->> group-clusters + (->> group-clusters ; XXX - Base clusters may not be necessary if we use the already computed bucket vote stats ; A vector, where each entry is a column iterator for the matrix subset of the given group - (mapv (comp mat/columns - nm/get-matrix - (partial nm/rowname-subset data) - #(clusters/group-members % base-clusters))) - (apply + (mapv (comp matrix/columns + nm/get-matrix + (partial nm/rowname-subset data) + #(clusters/group-members % base-clusters))) + (apply map (fn [& vote-cols-for-groups] (->> - (mapv comment-stats vote-cols-for-groups) - (utils/mapv-rest add-comparitive-stats)))))}) + (mapv comment-stats vote-cols-for-groups) + (utils/mapv-rest add-comparitive-stats)))))}) (defn- beats-best-by-test? "Returns true if a given comment/group stat hash-map has a more representative z score than current-best-z. Used for making sure we have at least _one_ representative comment for every group, even if none are left over after the more thorough filters." - [{:keys [rat rdt] :as comment-conv-stats} current-best-z] + [{:keys [rat rdt]} current-best-z] (or (nil? current-best-z) (> (max rat rdt) current-best-z))) @@ -140,7 +127,7 @@ "Like beats-best-by-test?, but only considers agrees. Additionally, doesn't focus solely on repness, but also on raw probability of agreement, so as to ensure that there is some representation of what people in the group agree on. Also, note that this takes the current-best, instead of just current-best-z." - [{:keys [na nd ra rat pa pat ns] :as comment-conv-stats} current-best] + [{:keys [na nd ra rat pa pat _ns]} current-best] (cond ; Explicitly don't let something that hasn't been voted on at all come into repness (= 0 na nd) @@ -151,18 +138,18 @@ (> (* ra rat pa pat) (apply * (map current-best [:ra :rat :pa :pat]))) ; If we have current-best, but only by prob estimate, just shoot for something that is generally agreed upon current-best - (> (* pa pat) (apply * (map current-best [:pa :pat]))) + (> (* pa pat) (apply * (map current-best [:pa :pat]))) ; Otherwise, accept if either repness or probability look generally good :else - (or (stats/z-sig-90? pat) - (and (> ra 1.0) - (> pa 0.5))))) + (or (stats/z-sig-90? pat) + (and (> ra 1.0) + (> pa 0.5))))) (defn- passes-by-test? "Decide whether we should count a given comment-conv-stat hash-map as being representative." ;; Should set the significance as a input variable or env variable XXX - [{:keys [pat rat pdt rdt] :as comment-conv-stats}] + [{:keys [pat rat pdt rdt]}] (or (and (stats/z-sig-90? rat) (stats/z-sig-90? pat)) (and (stats/z-sig-90? rdt) (stats/z-sig-90? pdt)))) @@ -170,7 +157,7 @@ (defn finalize-cmt-stats "Formats comment/repness stats to be consumable by clients. In particular, chooses between agree/disagree as to which is more representative, and populates a more regular structure accordingly." - [tid {:keys [na nd ns pa pd pat pdt ra rd rat rdt] :as comment-conv-stats}] + [tid {:keys [na nd ns pa pd pat pdt ra rd rat rdt]}] (let [[n-success n-trials p-success p-test repness repness-test repful-for] (if (> rat rdt) [na ns pa pat ra rat :agree] @@ -193,8 +180,8 @@ (defn repness-sort [repdata] (sort-by - (comp - repness-metric) - repdata)) + (comp - repness-metric) + repdata)) (defn agrees-before-disagrees @@ -202,87 +189,87 @@ [repdata] (concat ; Need vector so into appends, not prepends - (filter #(= :agree (:repful-for %)) repdata) - (filter #(= :disagree (:repful-for %)) repdata))) + (filter #(= :agree (:repful-for %)) repdata) + (filter #(= :disagree (:repful-for %)) repdata))) (defn select-rep-comments "Selects representative comments based on beats-best-by-test? and passes-by-test?. Always ensures there is at least one representative comment for a given cluster. Takes the results of conv-repness and returns a map of group cluster ids to representative comments and stats." - [{:keys [ids tids stats] :as repness-stats} & [mod-out]] + [{:keys [ids tids stats]} & [mod-out]] ; Reduce statistics into a results hash mapping group ids to rep comments (->> ; reduce with indices, so we have tids - (utils/zip tids stats) + (utils/zip tids stats) ; Apply moderation - (remove (comp (set mod-out) first)) - (reduce - (fn [result [tid stats]] + (remove (comp (set mod-out) first)) + (reduce + (fn [result [tid stats]] ; Inner reduce folds data into result for each group in comment stats ; XXX - could this be an assoc-in? - (reduce - (fn [group-result [gid comment-conv-stats]] + (reduce + (fn [group-result [gid comment-conv-stats]] ; Heplper functions for building our result; abbrv. gr = group-result - (letfn [(gr-get [gr & ks] - (get-in gr (into [gid] ks))) - (gr-assoc [gr & ks-and-val] - (assoc-in gr (into [gid] (butlast ks-and-val)) (last ks-and-val)))] - (as-> group-result gr + (letfn [(gr-get [gr & ks] + (get-in gr (into [gid] ks))) + (gr-assoc [gr & ks-and-val] + (assoc-in gr (into [gid] (butlast ks-and-val)) (last ks-and-val)))] + (as-> group-result gr ; First check to see if the comment data passes, and add if it does - (if (passes-by-test? comment-conv-stats) - (->> comment-conv-stats - (finalize-cmt-stats tid) - (conj (gr-get gr :sufficient)) - (gr-assoc gr :sufficient)) - gr) + (if (passes-by-test? comment-conv-stats) + (->> comment-conv-stats + (finalize-cmt-stats tid) + (conj (gr-get gr :sufficient)) + (gr-assoc gr :sufficient)) + gr) ; Keep track of what the best comment so far is, even if it doesn't pass, so we always have at ; least one comment - (if (and (empty? (gr-get gr :sufficient)) - (beats-best-by-test? comment-conv-stats (gr-get gr :best :repness-test))) - (gr-assoc gr :best (finalize-cmt-stats tid comment-conv-stats)) - gr) + (if (and (empty? (gr-get gr :sufficient)) + (beats-best-by-test? comment-conv-stats (gr-get gr :best :repness-test))) + (gr-assoc gr :best (finalize-cmt-stats tid comment-conv-stats)) + gr) ; Also keep track of best agree comment is, so we can throw that the front preferentially - (if (beats-best-agr? comment-conv-stats (gr-get gr :best-agree)) - (gr-assoc gr :best-agree (assoc comment-conv-stats :tid tid)) - gr)))) - result - (utils/zip ids stats))) + (if (beats-best-agr? comment-conv-stats (gr-get gr :best-agree)) + (gr-assoc gr :best-agree (assoc comment-conv-stats :tid tid)) + gr)))) + result + (utils/zip ids stats))) ; initialize result hash - (into {} (map #(vector % {:best nil :best-agree nil :sufficient []}) ids))) + (into {} (map #(vector % {:best nil :best-agree nil :sufficient []}) ids))) ; If no sufficient, use best; otherwise sort sufficient and take 5 - (map-vals - (fn [{:keys [best best-agree sufficient]}] - (let [best-agree (when best-agree + (map-vals + (fn [{:keys [best best-agree sufficient]}] + (let [best-agree (when best-agree ; finalize, and assoc in a :best-agree attribute, so the client can render ; accordingly - (assoc (finalize-cmt-stats (:tid best-agree) best-agree) - :n-agree (:na best-agree) - :best-agree true)) + (assoc (finalize-cmt-stats (:tid best-agree) best-agree) + :n-agree (:na best-agree) + :best-agree true)) ; Use best agree if that's what we have; otherwise best (possibly disagree); otherwise nothing - best-head (cond - best-agree [best-agree] - best [best] - :else [])] + best-head (cond + best-agree [best-agree] + best [best] + :else [])] ; If there weren't any matches of the criteria, just take the best match, and take the best agree if ; possible, and if not just the best general - (if (empty? sufficient) - best-head - (->> sufficient + (if (empty? sufficient) + best-head + (->> sufficient ; Remove best agree if in list, since we'll be putting it in manually; don't need to do this ; with best, since in any case we'd let it sort itself out - (remove #(= (:tid best-agree) (:tid %))) - (repness-sort) ; sort by our complicated repness metric - (concat (if best-agree [best-agree] [])) ; put best agree at front, if possible - (take 5) ; take the top 5 - (agrees-before-disagrees)))))))) + (remove #(= (:tid best-agree) (:tid %))) + (repness-sort) ; sort by our complicated repness metric + (concat (if best-agree [best-agree] [])) ; put best agree at front, if possible + (take 5) ; take the top 5 + (agrees-before-disagrees)))))))) (defn consensus-stats [data] (->> data nm/get-matrix - mat/columns + matrix/columns (map comment-stats) (map #(assoc %2 :tid %1) (nm/colnames data)))) @@ -292,20 +279,20 @@ (let [stats (->> cons-stats (remove (comp (set mod-out) :tid)) (map - (fn [stat] - (assoc stat - :dm (* (:pd stat) (:pdt stat)) - :am (* (:pa stat) (:pat stat)))))) + (fn [stat] + (assoc stat + :dm (* (:pd stat) (:pdt stat)) + :am (* (:pa stat) (:pat stat)))))) format-stat - (fn [cons-for stat] - (let [[n-success p-success p-test] (if (= cons-for :agree) - [:na :pa :pat] - [:nd :pd :pdt])] - {:tid (:tid stat) - :n-success (n-success stat) - :n-trials (:ns stat) - :p-success (p-success stat) - :p-test (p-test stat)})) + (fn [cons-for stat] + (let [[n-success p-success p-test] (if (= cons-for :agree) + [:na :pa :pat] + [:nd :pd :pdt])] + {:tid (:tid stat) + :n-success (n-success stat) + :n-trials (:ns stat) + :p-success (p-success stat) + :p-test (p-test stat)})) top-5 (fn [cons-for] (let [[metric prob test] (if (= cons-for :agree) [:am :pa :pat] @@ -369,13 +356,13 @@ (defn closeness [ptpt-position center] (- 1 - (mat/length - (- ptpt-position center)))) + (matrix/length + (- ptpt-position center)))) (defn extremeness [ptpt-position group-center group-extreme-direction] - (mat/dot (- ptpt-position group-center) - group-extreme-direction)) + (matrix/dot (- ptpt-position group-center) + group-extreme-direction)) (defn participant-stats [group-clusters base-clusters proj-nmat ptpt-vote-counts] @@ -389,7 +376,7 @@ size (count pids) ; group size ;; XXX Oh... actually we already have what we need to do the group centers from base; Should do that for perf center (mat-center (nm/rowname-subset proj-nmat pids)) - extreme-direction (mat/normalise (- center global-center))] + extreme-direction (matrix/normalise (- center global-center))] [id {:gid id :pids pids :center center :size size :extreme-direction extreme-direction}]))) (into {})) ;; Compute the global center diff --git a/math/src/polismath/math/stats.clj b/math/src/polismath/math/stats.clj index 17399aa722..f133d5f2d4 100644 --- a/math/src/polismath/math/stats.clj +++ b/math/src/polismath/math/stats.clj @@ -1,17 +1,15 @@ ;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . (ns polismath.math.stats - ;; XXX Moe to require - (:require [plumbing.core :as pc] - [clojure.core.matrix :as matrix] - [taoensso.timbre :as log])) + (:require + [clojure.math :as math])) (defn prop-test [succ n] (let [[succ n] (map inc [succ n])] (* 2 - (matrix/sqrt n) + (math/sqrt n) (+ (/ succ n) -0.5)))) @@ -27,10 +25,10 @@ ; that lets you take the ratio of the derivatives or something... 0 (/ (- pi1 pi2) - (matrix/sqrt - (* pi-hat - (- 1 pi-hat) - (+ (/ 1 pop-in) (/ 1 pop-out)))))))) + (math/sqrt + (* pi-hat + (- 1 pi-hat) + (+ (/ 1 pop-in) (/ 1 pop-out)))))))) (defn z-sig-90? @@ -39,11 +37,5 @@ (> z-val 1.2816)) -(defn z-sig-95? - "Test whether z-statistic is significant at %95 confidence with alternative hypothesis >" - [z-val] - (> z-val 1.6449)) - - :ok diff --git a/math/src/polismath/meta/metrics.clj b/math/src/polismath/meta/metrics.clj index 12799871df..2fca558693 100644 --- a/math/src/polismath/meta/metrics.clj +++ b/math/src/polismath/meta/metrics.clj @@ -5,9 +5,6 @@ [com.stuartsierra.component :as component])) -;; Should maybe make this component depend upon mongo so that it handles both metrics that get to mongo and -;; those sent to graphitedb. Either that or we should rename the component and namespace graphitedb... - (defn- make-socket "Make a datagram socket; optional port parameter is the local port for the socket. If ommitted (or if nil is passed), the Java implementation will pick some available port and bind it." @@ -38,9 +35,9 @@ (.send send-socket send-packet))) (defn- make-send-string - "All keys are name keys are prepended with 'math..'..." - [math-env api-key values] - (str api-key ".math." math-env "." (partial clojure.string/join " " values) \n)) + "All metric keys are prepended with 'math.prod.'" + [api-key values] + (str api-key ".math.prod." (partial clojure.string/join " " values) \n)) (defn- send-metric-values [metric-sender values] @@ -48,8 +45,7 @@ (when (and api-key hostname) ; Only send metrics if properly configured (log/info "sending metric data " values " to " hostname ":" remote-port) (send-data metric-sender - (make-send-string (-> metric-sender :config :math-env) - api-key + (make-send-string api-key values))))) ;; ## Public API @@ -72,19 +68,5 @@ (log/debug (str end# " " ~metric-name " " duration# " millis")) ret#)) -;; It looks like we can optionally get responses from graphitedb? We're not using that capability for now -;; though. - -;(defn receive-data [receive-socket] - ;(let [receive-data (byte-array 1024), - ;receive-packet (new java.net.DatagramPacket receive-data 1024)] - ;(.receive receive-socket receive-packet) - ;(new java.lang.String (.getData receive-packet) 0 (.getLength receive-packet)))) - - -;(defn make-receive [receive-port] - ;(let [receive-socket (make-socket receive-port)] - ;(fn [] (receive-data receive-socket)))) - :ok diff --git a/math/src/polismath/meta/microscope.clj b/math/src/polismath/meta/microscope.clj index 7face2d0c9..e79c75e285 100644 --- a/math/src/polismath/meta/microscope.clj +++ b/math/src/polismath/meta/microscope.clj @@ -1,20 +1,11 @@ ;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . (ns polismath.meta.microscope - (:require [polismath.conv-man :as cm] - [polismath.components.env :as env] - [polismath.components.postgres :as db] - [polismath.math.conversation :as conv] - [polismath.math.named-matrix :as nm] - [polismath.utils :as utils] - [clojure.tools.trace :as tr] - [clojure.tools.logging :as log] - [clojure.tools.cli :refer [parse-opts]] - [com.stuartsierra.component :as component] - [plumbing.core :as pc] - [korma.core :as ko] - [korma.db :as kdb] - [polismath.components.postgres :as postgres])) + (:require + [clojure.tools.logging :as log] + [polismath.components.postgres :as db] + [polismath.conv-man :as cm] + [polismath.utils :as utils])) ; ; ; @@ -30,59 +21,14 @@ (defn recompute [{:as system :keys [conversation-manager postgres]} & {:keys [zid zinvite] :as args}] (assert (utils/xor zid zinvite)) - (let [zid (or zid (postgres/get-zid-from-zinvite (:postgres system) zinvite)) + (let [zid (or zid (db/get-zid-from-zinvite (:postgres system) zinvite)) new-votes (db/conv-poll postgres zid 0)] (log/info "Running a recompute on zid:" zid "(zinvite:" zinvite ")") (cm/queue-message-batch! conversation-manager :votes zid new-votes))) (comment (require '[polismath.runner :as runner :refer [system]]) - (postgres/get-zid-from-zinvite (:postgres system) "7scufp") + (db/get-zid-from-zinvite (:postgres system) "7scufp") (recompute system :zinvite "7scufp") :end-example-comment) - -(defn kw->int - [kw] - (-> kw - (str) - (clojure.string/replace ":" "") - (Integer/parseInt))) - - - -;(defn replay-conv-update -; "Can be run as a shell command on a error file to replay what happened." -; [filename] -; (let [data (conv/load-conv-update filename) -; {:keys [conv votes opts]} data -; {:keys [rating-mat base-clusters pca]} conv] -; (println "Loaded conv:" filename) -; (println "Dimensions:" (count (nm/rownames rating-mat)) "x" (count (nm/colnames rating-mat))) -; (conv/conv-update conv votes))) -; -; -;(def cli-options -; [["-z" "--zid ZID" "ZID on which to do a rerun" :parse-fn #(Integer/parseInt %)] -; ["-Z" "--zinvite ZINVITE" "ZINVITE code on which to perform a rerun"] -; ["-r" "--recompute" "If set, will run a full recompute"]]) -; -; -;(defn -main -; [& args] -; (let [{:keys [options arguments errors summary]} (parse-opts args cli-options) -; conv-actor (utils/apply-kwargs recompute options) -; done? (atom false)] -; (add-watch -; (:conv conv-actor) -; :shutdown-watch -; (fn [k r o n] -; (println n) -; (swap! done? (fn [_] true)) -; (shutdown-agents))) -; (loop [] -; (Thread/sleep 1000) -; (when-not @done? -; (recur))))) -; - diff --git a/math/src/polismath/meta/notify.clj b/math/src/polismath/meta/notify.clj index f2e8e7cb2c..9b648a448d 100644 --- a/math/src/polismath/meta/notify.clj +++ b/math/src/polismath/meta/notify.clj @@ -1,33 +1,6 @@ ;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . -(ns polismath.meta.notify - (:require [clj-http.client :as client] - [taoensso.timbre :as log])) - - -;; Keep track of the type of notifications sent, and when they were sent, so you can avoid spamming. -;; Maps from message type to a list of timestamps. If there are more than N timestamps within T seconds, it prevents sending. Timestamps older than T seconds will be cleared. -(def team-notifications-per-message-type-throttle (atom {})) -;; Maps from zid,message-type to time last sent in millis. -(def team-notifications-per-convo (atom {})) - - -(def throttle-count 5) ; number of messages of message-type that can be sent during a throttle-time amount of time -(def throttle-time (* 60 60 1000)) -(def min-wait (* 60 60 1000)) - -(defn send-team-notification [config subject body] - (try - (let [response (client/post (str (:webserver-url config) "/notifyTeam") - {:form-params {:webserver_username (:webserver-username config) - :webserver_pass (:webserver-pass config) - :subject subject - :body body} - :content-type :json - :throw-entire-message? true})] - (log/info "notifyTeam response:\n" (with-out-str (clojure.pprint/pprint response)))) - (catch Exception e - (log/error e "failed to notifyTeam:\n")))) +(ns polismath.meta.notify) (defn error-message-body @@ -40,20 +13,4 @@ (for [row trace] (str " " row "\n")))))) -(defn notify-team [config message-type zid subject body] - (let [now (quot (System/currentTimeMillis) 1000) - per-convo-key [zid message-type] - oldest (- now throttle-time) - trimmed-throttle-list (filter (fn [t] (> t oldest)) (get @team-notifications-per-message-type-throttle message-type)) - message-type-throttle-count (count trimmed-throttle-list)] - (if (and - (< message-type-throttle-count throttle-count) ; don't send more than 5 per hour for a given message type - (or ; don't send more than one per zid,message-type per hour - (not (contains? @team-notifications-per-convo per-convo-key)) - (>= now (+ min-wait (get @team-notifications-per-convo per-convo-key))))) - (do - (send-team-notification config subject body) - (swap! team-notifications-per-convo (fn [old] (assoc old per-convo-key now))) - (swap! team-notifications-per-message-type-throttle (fn [old] (assoc old message-type (conj trimmed-throttle-list now)))))))) - :ok diff --git a/math/src/polismath/meta/simulation.clj b/math/src/polismath/meta/simulation.clj deleted file mode 100644 index 94fa7cdef5..0000000000 --- a/math/src/polismath/meta/simulation.clj +++ /dev/null @@ -1,270 +0,0 @@ -;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . - -(ns polismath.meta.simulation) -; (:require [polismath.math.conversation :as conversation] -; [polismath.math.named-matrix :as nm] -; [polismath.utils :as utils] -; [clojure.tools.cli :refer [parse-opts]] -; [clojure.string :as string] -; [plumbing.core :as pc] -; [bigml.sampling [reservoir :as reservoir] -; [simple :as simple]] -; [clj-time.coerce :as time-coerce] -; [clj-time.local :as time-local])) -; -; -;(defprotocol Voteable -; (cast-vote! [this] [this cmnt] [this member cmnt])) -; -; -;(defrecord CommentVoteDist -; [p-agree p-disagree] -; Voteable -; (cast-vote! [_] -; (let [r (rand)] -; (cond -; (> r p-agree) -1 -; (> r (+ p-agree p-disagree)) 1 -; :else 0)))) -; -; -;(defn comment-vote-dist! -; "A somewhat bimodal random vote distribution generator. Could be more principaled about this, but I think -; it's fine for starters." -; ([] -; (let [p1 (rand) -; p2 (rand (- 1 p1)) -; p1-for-aggree (= (rand-int 2) 0) -; p-agree (if p1-for-aggree p1 p2) -; p-disagree (if p1-for-aggree p2 p1)] -; (CommentVoteDist. p-agree p-disagree))) -; ([p-agree] -; (let [p-disagree (rand (- 1 p-agree))] -; (CommentVoteDist. p-agree p-disagree))) -; ([p-agree p-disagree] -; (assert (<= (+ p-agree p-disagree) 1)) -; (CommentVoteDist. p-agree p-disagree))) -; -; -;(defrecord Group -; [comment-dists] -; Voteable -; (cast-vote! [_ cmnt] -; (cast-vote! (get comment-dists cmnt)))) -; -; -;(defn new-group [] -; (Group. [])) -; -; -;(defn grp-add-cmnt -; [grp & {:keys [dist]}] -; (update-in grp [:comment-dists] conj (or dist (comment-vote-dist!)))) -; -; -;(defn random-grp-i -; [grp-dists] -; (first -; (simple/sample (range (count grp-dists)) -; :weight grp-dists))) -; -; -;(defn add-member -; [conv & [grp-i]] -; (let [member (count (:members conv))] -; (-> conv -; (update-in [:members] conj (or grp-i (random-grp-i (:grp-dists conv)))) -; (update-in [:unvoted] into (for [c (range (:comments conv))] [member c]))))) -; -; -;(defn conv-add-cmnt -; [conv] -; (let [cmnt (:comments conv)] -; (-> conv -; (update-in [:groups] (partial mapv grp-add-cmnt)) -; (update-in [:unvoted] into (for [m (range (count (:members conv)))] [m cmnt])) -; (update-in [:comments] inc)))) -; -; -;(defn fn-exp -; [f n] -; (apply comp (repeat n f))) -; -; -;(defn add-members -; [conv n] -; ((fn-exp add-member n) conv)) -; -; -;(defn add-cmnts -; [conv n] -; ((fn-exp conv-add-cmnt n) conv)) -; -; -;(defn sim-conv -; [& {:keys [zid n-grps grp-dists n-ptpts n-cmnts] :or {n-grps 3 n-ptpts 0 n-comnts 0}}] -; (-> {:zid (or zid (rand-int 1000)) -; :groups (repeat n-grps (new-group)) -; :unvoted [] -; :members [] ; actually a map of member indices to group indices -; :comments 0 -; :grp-dists (or grp-dists (repeat n-grps (/ 1 n-grps)))} -; (add-members n-ptpts) -; (add-cmnts n-cmnts))) -; -; -;(defn conv-votes -; [conv & [n]] -; (let [n (or n 1)] -; (let [zid (:zid conv) -; picks (reservoir/sample (:unvoted conv) n) -; new-conv (update-in conv [:unvoted] (partial remove (set picks))) -; ; Handle the situation where we want more votes than there are :unvoted (ptpt, cmnt) pairs by having -; ; revotes -; picks (if (= (count picks) n) -; picks -; (concat picks (for [_ (range (- n (count picks)))] -; [(rand-int (count (:members conv))) -; (rand-int (:comments conv))]))) -; votes (map -; (fn [[m c]] -; (let [grp-i (get-in conv [:members m]) -; grp ((:groups conv) grp-i)] -; {:zid zid :pid m :tid c :vote (cast-vote! grp c)})) -; picks)] -; ; Return the new conversation state so it can be looped on -; [new-conv votes]))) -; -; -;(defprotocol Pollable -; (poll! [this state last-vote-timestamp])) -; -; -;(defn unzip -; [xys] -; (reduce -; (fn [[xs ys] xy] -; [(conj xs (first xy)) (conj ys (second xy))]) -; [[] []] -; xys)) -; -; -;(defn rand-ms -; [max] -; (+ (* (rand-int (int (/ max 1000))) -; 1000) -; (rand-int 1000))) -; -; -; -;(defn add-timestamps -; [[conv votes] last-vote-timestamp] -; (let [old-last-ts (or last-vote-timestamp 0) -; now (System/currentTimeMillis) -; ts-diff (- now old-last-ts)] -; [conv -; (map #(assoc % :created (+ (rand-ms ts-diff) old-last-ts)) votes)])) -; -; -;(defn new-poller -; "Takes a collection of conversation simulators, and a set of growth functions, each of which takes an argument of -; the conversation sims, and uses infomation there to decide how to modify the conversation and how to poll" -; [& {:keys [ptpt-growth-fn -; cmnt-growth-fn -; poll-count-fn]}] -; (reify -; Pollable -; (poll! [this conv last-vote-timestamp] -; (let [[new-ptpts new-cmnts n-votes] (map #(% conv) [ptpt-growth-fn cmnt-growth-fn poll-count-fn])] -; (-> conv -; ((fn-exp add-member new-ptpts)) -; ((fn-exp conv-add-cmnt new-cmnts)) -; (conv-votes n-votes) -; ; Adds :created timestamps -; (add-timestamps last-vote-timestamp)))))) -; -; -;(def simple-poller -; (new-poller -; :ptpt-growth-fn (fn [conv] 10) -; :cmnt-growth-fn (fn [conv] 1) -; :poll-count-fn (fn [conv] -; (* (inc (count (:members conv))) 2)))) -; -; -;(defn comp-poller -; [& pollers] -; (reify -; Pollable -; (poll! [this convs last-vote-timestamp] -; (let [[new-convs vote-batches] -; (unzip -; (map #(poll! %1 %2 last-vote-timestamp) pollers convs))] -; [new-convs -; (->> vote-batches -; (apply concat) -; (sort-by :created))])))) -; -; -;(defn int-opt [& args] -; (conj (into [] args) -; :parse-fn #(Integer/parseInt %))) -; -; -;(def cli-options -; [(int-opt "-i" "--poll-interval INTERVAL" "Milliseconds between randomly generated polls" :default 1000) -; (int-opt "-z" "--n-convs NUMBER" "Number of conversations to simulate" :default 3) -; (int-opt "-p" "--person-count-start COUNT" :default 4) -; (int-opt "-P" "--person-count-growth COUNT" :default 3) -; (int-opt "-c" "--comment-count-start COUNT" :default 3) -; (int-opt "-C" "--comment-count-growth COUNT" :default 1) -; ["-h" "--help"]]) -; -; -;(defn usage [options-summary] -; (->> ["Polismath simulations" -; "Usage: lein run -m polismath.simulation [options]" -; "" -; "Options:" -; options-summary] -; (string/join \newline))) -; -; -;;; Just commenting out the redis/carmine specific stuff -; -;;; See `wcar` docstring for opts -;;(def server1-conn {:pool {} -; ;:spec {}}) -;;(defmacro wcar* [& body] `(car/wcar server1-conn ~@body)) -;;(def wcar-worker* (partial car-mq/worker server1-conn)) -; -;;(defn simulate! -; ;[{:keys [n-convs poll-interval]}] -; ;(let [pollers (repeat n-convs simple-poller) -; ;poller (apply comp-poller pollers)] -; ;(loop [convs (for [i (range n-convs)] -; ;(sim-conv :n-ptpts 4 :n-cmnts 5 :zid i)) -; ;last-vote-timestamp 0 -; ;polls 0] -; ;(Thread/sleep poll-interval) -; ;(let [[new-convs votes] (poll! poller convs last-vote-timestamp) -; ;new-last-vote-timestamp (apply max (map :created votes))] -; ;(println "Simulating" (count votes)) -; ;(wcar* (car-mq/enqueue "simvotes" (vec votes))) -; ;(recur new-convs new-last-vote-timestamp (inc polls)))))) -; -;;(comment -; ;(simulate! {:n-convs 3 :poll-interval 3}) -; ;) -; -; -;;;; XXX Move to a system -;;(defn -main [& args] -; ;(println "Starting simulations") -; ;(let [{:keys [options arguments errors summary]} (parse-opts args cli-options)] -; ;(cond -; ;(:help options) (exit 0 (usage summary)) -; ;(:errors options) (exit 1 (str "Found the following errors:" \newline (:errors options))) -; ;:else (simulate! options)))) -; - diff --git a/math/src/polismath/poller.clj b/math/src/polismath/poller.clj index 2830c1da4b..b35f5c5727 100644 --- a/math/src/polismath/poller.clj +++ b/math/src/polismath/poller.clj @@ -1,16 +1,16 @@ ;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . (ns polismath.poller - (:require [clojure.core.async :as async :refer [go go-loop >! !! ! go go-loop]] + [com.stuartsierra.component :as component] + [polismath.components.postgres :as postgres] + [polismath.conv-man :as conv-man] + [taoensso.timbre :as log])) (defn poll - [{:as poller :keys [config conversation-manager postgres kill-chan]} message-type] + [{:keys [config conversation-manager postgres kill-chan]} message-type] (let [poller-config (-> config :poller) start-polling-from (- (System/currentTimeMillis) (* (:poll-from-days-ago poller-config) 1000 60 60 24)) polling-interval (or (-> poller-config (get message-type) :polling-interval) 1000) diff --git a/math/src/polismath/runner.clj b/math/src/polismath/runner.clj index 18ed78e812..aa5ca6203c 100644 --- a/math/src/polismath/runner.clj +++ b/math/src/polismath/runner.clj @@ -3,37 +3,32 @@ (ns polismath.runner "This namespace is responsible for running systems" (:refer-clojure :exclude [run!]) - (:require [polismath.system :as system] - ;[polismath.stormspec :as stormspec :refer [storm-system]] - [polismath.utils :as utils] - ;; TODO Replace this with the canonical clojure.tools.cli once storm is removed - [clojure.tools.cli :as cli] - [clojure.tools.namespace.repl :as namespace.repl] - [taoensso.timbre :as log] - [clojure.string :as string] - [clj-time.core :as t] - [clojure.edn :as edn] - [clj-time.coerce :as co] - [clojure.java.io :as io] - [com.stuartsierra.component :as component] - [polismath.darwin.export :as export] - [polismath.conv-man :as conv-man] - [polismath.math.conversation :as conv] - [polismath.components.postgres :as postgres] - [clojure.pprint :as pprint]) - (:import [java.util.zip ZipOutputStream ZipEntry])) + (:require + [clj-time.coerce :as co] + [clj-time.core :as t] + [clojure.java.io :as io] + [clojure.pprint :as pprint] + [clojure.string :as string] + [clojure.tools.cli :as cli] + [clojure.tools.namespace.repl :as namespace.repl] + [com.stuartsierra.component :as component] + [polismath.components.postgres :as db] + [polismath.conv-man :as conv-man] + [polismath.darwin.export :as export] + [polismath.math.conversation :as conv] + [polismath.system :as system] + [polismath.utils :as utils] + [taoensso.timbre :as log]) + (:import + [java.util.zip ZipOutputStream])) (defonce system nil) -;(def system nil) - -;; Should build this to be an atom, and build something that intiates this state from a config file. -;; So when you reset, it reboots all systems. (defn init! ([system-map-generator config-overrides] (alter-var-root #'system - (constantly (utils/apply-kwargs component/system-map (system-map-generator config-overrides))))) + (constantly (utils/apply-kwargs component/system-map (system-map-generator config-overrides))))) ([system-map-generator] (init! system-map-generator {}))) @@ -45,7 +40,7 @@ (defn stop! [] (alter-var-root #'system - (fn [s] (when s (component/stop s))))) + (fn [s] (when s (component/stop s))))) (defn run! ([system-map-generator config-overrides] @@ -59,28 +54,23 @@ (run! system/poller-system))) (defn system-reset! - ([] - (stop!) - (namespace.repl/refresh :after 'polismath.runner/run!))) + "Stop the system, reload all changed namespaces, and restart. + This is a convenience function for REPL-driven development. + See also: `stop!`, `start!`, `run!`" + [] + (stop!) + (namespace.repl/refresh :after 'polismath.runner/run!)) (def subcommands - {;"storm" stormspec/storm-system ;; remove... - ;"onyx" system/onyx-system ;; soon... - "update-all" system/base-system + {"update-all" system/base-system "update" system/base-system "poller" system/poller-system "tasks" system/task-system "full" system/full-system - "simulator" system/simulator-system "export" system/export-system}) -;; TODO Build nice cli settings forking on subcommand, and tie in sanely with options comp - -;; QUESTION How do we fork things nicely on command that get run and exit, vs long lived? - (def cli-options - "Has the same options as simulation if simulations are run" [["-r" "--recompute" "Recompute conversations from scratch instead of starting from most recent values"] ["-h" "--help" "Print help and exit"] ["-z" "--zid ZID" "ZID on which to do an export" :parse-fn #(Integer/parseInt %)] @@ -94,7 +84,7 @@ "" "Other options:" options-summary] - (string/join \newline))) + (string/join \newline))) @@ -103,18 +93,18 @@ (try (let [conv (conv-man/load-or-init conv-man zid) updated-conv (conv/conv-update conv []) - math-tick (postgres/inc-math-tick (:postgres conv-man) zid)] + math-tick (db/inc-math-tick (:postgres conv-man) zid)] (conv-man/write-conv-updates! conv-man updated-conv math-tick)) (catch Exception e (log/error e (str "Unable to complete conversation update for zid " zid))))) (defn update-all-convs - [{:as system :keys [conversation-manager postgres]}] + [{:keys [conversation-manager postgres]}] (->> - (postgres/ptpt-counts postgres) - (map :ptpt_cnt) - (pmap (partial update-conv conversation-manager)) - (doall))) + (db/ptpt-counts postgres) + (map :ptpt_cnt) + (pmap (partial update-conv conversation-manager)) + (doall))) @@ -152,23 +142,11 @@ ["-f" "--filename FILENAME" "filename" "Name of output file (should be zip for csv out)"] ["-t" "--at-time AT_TIME" "A string of YYYY-MM-DD-HH-MM-SS (in UTC) or ms-timestamp since epoch" :parse-fn parse-time] ["-T" "--at-times AT_TIMES" "A vector of strings of --at-time format" :parse-fn parse-times] - ["-F" "--format FORMAT" "Either csv, excel or (soon) json" :parse-fn keyword :validate [#{:csv :excel} "Must be either csv or excel"] :default :csv] + ["-F" "--format FORMAT" "Either csv or json" :parse-fn keyword :validate [#{:csv :json} "Must be either csv or json"] :default :csv] ["-M" "--update-math" "Update math"] ["-P" "--update-postgres" "Update postgres"] ["-h" "--help" "Print help and exit"]]) -(defn error-msg [errors] - (str "The following errors occurred while parsing your command:\n\n" - (clojure.string/join \newline errors))) - -(defn help-msg [options] - (str "Export a conversation or set of conversations according to the options below:\n\n" - \tab - "filename" \tab "Filename (or file basename, in case of zip output, implicit or explicit" \newline - (clojure.string/join \newline - (for [opt cli-options] - (apply str (interleave (repeat \tab) (take 3 opt))))))) - (defn run-export [system {:as options :keys [filename user-id at-times] @@ -185,24 +163,24 @@ (let [zinvite (export/get-zinvite-from-zid darwin zid)] (log/info "Now working on conv:" zid zinvite) (export/export-conversation darwin - (assoc options - :zid zid - :zip-stream zip - :writer writer - :entry-point (str (export/zipfile-basename filename) "/" zinvite)))))) + (assoc options + :zid zid + :zip-stream zip + :writer writer + :entry-point (str (export/zipfile-basename filename) "/" zinvite)))))) ;; Want to print the status of the conversation at each time t in at-times at-times (with-open [file (io/output-stream filename) zip (ZipOutputStream. file) writer (io/writer zip)] (doseq [at-time at-times] - (log/info "Now working on conv:" (:zid options) (:zinvite options)) - (export/export-conversation darwin - (assoc options - :at-time at-time - :zip-stream zip - :writer writer - :entry-point (str (export/zipfile-basename filename) "/" at-time))))) + (log/info "Now working on conv:" (:zid options) (:zinvite options)) + (export/export-conversation darwin + (assoc options + :at-time at-time + :zip-stream zip + :writer writer + :entry-point (str (export/zipfile-basename filename) "/" at-time))))) :else (export/export-conversation darwin options)) (utils/exit 0 "Export complete"))) @@ -213,7 +191,7 @@ ;; default to poller subcommand (let [subcommand (or (first args) "poller") parser-spec (if (= subcommand "export") export-cli-options cli-options) - {:as parse-results :keys [arguments options errors summary]} (cli/parse-opts args parser-spec)] + {:as parse-results :keys [options errors summary]} (cli/parse-opts args parser-spec)] (log/info "CLI arguments and options:\n" (with-out-str (pprint/pprint (select-keys parse-results [:arguments :options])))) (cond ;; Help message @@ -222,14 +200,13 @@ ;; Error in parsing (this should really catch the below condition as well errors (utils/exit 1 (apply str "Found the following errors:\n" - (clojure.string/join "\n" errors) "\n" - usage summary)) + (clojure.string/join "\n" errors) "\n" + usage summary)) ;; otherwise, run the thing :else (let [system-map-generator (subcommands subcommand) sys-options (if (#{"export"} subcommand) - ;; pretty much always want prod here, regardless of env variable; overrideable? - {:math-env :prod :export {:temp-dir "."}} + {:export {:temp-dir "."}} options) system (system/create-and-run-system! system-map-generator sys-options)] (case subcommand @@ -244,22 +221,5 @@ ;; Otherwise, default to keeping the main thread spinning while the system runs (loop [] (Thread/sleep 1000) - (recur))))))) - - -(comment - ;(run! system/poller-system) - (run! system/base-system) - - (conv-man/load-or-init) - - ;(require '[polismath.conv-man :as conv-man]) - ;(let [conv-man (:conversation-manager system)] - ;(conv-man/queue-message-batch! conv-man )) - - (stop!) - :endcomment) - - - + (recur))))))) diff --git a/math/src/polismath/stormspec.clj b/math/src/polismath/stormspec.clj deleted file mode 100644 index 4cb3afe3e6..0000000000 --- a/math/src/polismath/stormspec.clj +++ /dev/null @@ -1,142 +0,0 @@ -;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . - -(ns polismath.stormspec) - ;(:import [backtype.storm StormSubmitter LocalCluster]) - ;(:require [polismath.system :as system] - ; [polismath.conv-man :as cm] - ; [polismath.components.postgres :as postgres] - ; [polismath.components.env :as env] - ; [polismath.math.named-matrix :as nm] - ; [polismath.utils :as utils] - ; [polismath.math.conversation :as conv] - ; ;[polismath.simulation :as sim] - ; [clojure.core.matrix :as matrix] - ; [clojure.string :as string] - ; [clojure.tools.logging :as log] - ; [clojure.core.async :as async :refer [go ! !! alts!! alts! chan dropping-buffer put! take!]] - ; [com.stuartsierra.component :as component] - ; [backtype.storm [clojure :as storm] [config :as storm-config]] - ; [plumbing.core :as pc]) - ;;; I don't think we need this anymore on newer Clojures, so should remove XXX - ;(:gen-class)) - - -;; XXX Maybe loading the current implementation can be a system boot step? -; XXX - storm hack. Solves issue where one process or thread has started loading vectorz, but the other -; doesn't know to wait (at least this is what seems to be the case) -;(matrix/set-current-implementation :vectorz) -;(matrix/matrix [[1 2 3] [4 5 6]]) - - -(comment - (storm/defspout poll-spout ["message-type" "zid" "batch"] - ;; Should fork timestamp key on message-type as well XXX - {:params [system-config message-type timestamp-key] :prepare true} - [conf context collector] - ;(let [last-timestamp (atom 0) - ;;; Need config for storm to be set under keys of :vote and :mod, so the spouts are configurable - ;;; more simply from this; But I guess some of these things should maybe not be constructable this - ;;; way... - ;;system (-> system-config system/base-system component/system-map component/start) - ;system (system/create-and-run-base-system! system-config) - ;(storm/spout - (let [system (system/create-and-run-base-system! system-config) - polling-interval (-> system-config :storm :spouts message-type :polling-interval) - postgres (-> system :postgres) - last-timestamp (atom (try (java.lang.Long/parseLong (:initial-polling-timestamp env/env)) (catch Exception e (log/warn "INITIAL_POLLING_TIMESTAMP not set; setting to 0") 0)))] - ;(if (= poll-fn :sim-poll) - ;(log/info "Starting sim poller!") - ;(start-sim-poller!)) - (storm/spout - (nextTuple [] - (try - (log/info "Polling" message-type ">" @last-timestamp) - (let [poll-function (get {:votes postgres/poll :moderation postgres/mod-poll} message-type) - results (poll-function postgres @last-timestamp) - grouped-batches (group-by :zid results)] - ; For each chunk of votes, for each conversation, send to the appropriate spout - (doseq [[zid batch] grouped-batches] - (storm/emit-spout! collector [message-type zid batch])) - ; Update timestamp if needed - (swap! last-timestamp - (fn [last-ts] (apply max 0 last-ts (map timestamp-key results))))) - (catch Exception e - (log/error "Exception bubbled up in poll-spout!") - (.printStackTrace e))) - (Thread/sleep polling-interval)) - (ack [id])))) - - ;; XXX Should build a recompute trigger spout, which just sends a recompute message message-type with nil batch payload - - (storm/defbolt conv-update-bolt [] - {:params [system-config] :prepare true} - [conf context collector] - (let [system (system/create-and-run-base-system! system-config) - conv-man (:conversation-manager system)] - (storm/bolt - (execute [tuple] - (try - (let [[message-type zid batch] (.getValues tuple)] - ;; XXX Need to think more about recompute... - (cm/queue-message-batch! conv-man message-type zid batch)) - (storm/ack! collector tuple) - (catch Exception e - (log/error "Exception bubbled up in conv-update-bold!") - (.printStackTrace e))))))) - - (defn make-topology - ;; Note here that we're pushing through the full system configuration as config overrides, because we need - ;; something serializable for the nodes to boot their systems from in the prepare steps - [{:keys [config] :as storm-cluster}] - (let [config (into {} config) ;; O/w have a non-serializable object - spouts {"vote-spout" (storm/spout-spec (poll-spout config :votes :created)) - "mod-spout" (storm/spout-spec (poll-spout config :moderation :modified))} - bolt-inputs (into {} (for [s (keys spouts)] [s ["zid"]]))] - (assoc storm-cluster - :topology - (storm/topology - spouts - {"conv-update" (storm/bolt-spec bolt-inputs (conv-update-bolt config))})))) - - (defn submit-topology! - [storm-cluster] - (let [{:keys [execution cluster-name workers]} (-> storm-cluster :config :storm) - cluster-submitter (case execution - :local (LocalCluster.) - :distributed (StormSubmitter.))] - (.submitTopology cluster-submitter - cluster-name - {storm-config/TOPOLOGY-DEBUG false - storm-config/TOPOLOGY-WORKERS workers} - (:topology storm-cluster)) - (assoc storm-cluster :cluster-submitter cluster-submitter))) - - (defn stop-cluster! - [{:as storm-cluster :keys [cluster-submitter]}] - (let [cluster-name (-> storm-cluster :config :storm :cluster-name)] - ;(.killTopology topology cluster-name) - (.shutdown cluster-submitter))) - - - (defrecord StormCluster [config conversation-manager topology cluster-submitter] - component/Lifecycle - (start [component] - (-> component - make-topology - submit-topology!)) - (stop [component] - (stop-cluster! component) - (assoc component :topology nil))) - - - (defn create-storm-cluster [] - (map->StormCluster {})) - - - (defn storm-system - "Creates a base-system and assocs in polismath storm worker related components." - [config-overrides] - (merge (system/base-system config-overrides) - {:storm-cluster (component/using (create-storm-cluster) [:config :conversation-manager])}))) - - diff --git a/math/src/polismath/system.clj b/math/src/polismath/system.clj index afa511e231..08983de6bf 100644 --- a/math/src/polismath/system.clj +++ b/math/src/polismath/system.clj @@ -1,21 +1,18 @@ ;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . (ns polismath.system - (:require [polismath.utils :as utils] - [polismath.components [config :as config] - [postgres :as postgres] - [core-matrix-boot :as core-matrix-boot] - [logger :as logger]] - ;[server :as server]] - [polismath.conv-man :as conv-man] - [polismath.poller :as poller] - [polismath.tasks :as tasks] - [polismath.darwin.core :as darwin] - [taoensso.timbre :as log] - [clojure.tools.namespace.repl :as namespace.repl] - [clojure.string :as string] - [clojure.tools.cli :as cli] - [com.stuartsierra.component :as component])) + (:require + [com.stuartsierra.component :as component] + [polismath.components [config :as config] + [postgres :as postgres] + [core-matrix-boot :as core-matrix-boot] + [logger :as logger]] + [polismath.conv-man :as conv-man] + [polismath.darwin.core :as darwin] + [polismath.poller :as poller] + [polismath.tasks :as tasks] + [polismath.utils :as utils] + [taoensso.timbre :as log])) (defn base-system "This constructs an instance of the base system components, including config, db, etc." @@ -30,42 +27,28 @@ "Creates a base-system and assocs in darwin server related components." [config-overrides] (merge - (base-system config-overrides) - {:vote-poller (component/using (poller/create-poller :votes) [:config :postgres :conversation-manager :logger]) - :mod-poller (component/using (poller/create-poller :moderation) [:config :postgres :conversation-manager :logger])})) + (base-system config-overrides) + {:vote-poller (component/using (poller/create-poller :votes) [:config :postgres :conversation-manager :logger]) + :mod-poller (component/using (poller/create-poller :moderation) [:config :postgres :conversation-manager :logger])})) (defn task-system [config-overrides] (merge - (base-system config-overrides) - {:darwin (component/using (darwin/create-darwin) [:config :postgres :conversation-manager]) - :task-poller (component/using (tasks/create-task-poller) [:config :darwin :postgres :conversation-manager])})) + (base-system config-overrides) + {:darwin (component/using (darwin/create-darwin) [:config :postgres :conversation-manager]) + :task-poller (component/using (tasks/create-task-poller) [:config :darwin :postgres :conversation-manager])})) (defn export-system [config-overrides] (log/info "export system overrides:" config-overrides) (merge - (base-system config-overrides) - {:darwin (component/using (darwin/create-darwin) [:config :postgres :conversation-manager])})) + (base-system config-overrides) + {:darwin (component/using (darwin/create-darwin) [:config :postgres :conversation-manager])})) (defn full-system [config-overrides] (merge - (poller-system config-overrides))) - ;; This is a little silly to do this here, since we can just change the commands that get called to only - ;; run the poller system, but this is more expedient for the moment, and trying to get things smoothed out - ;; for developer meetup tomorrow :grimacing: - ;(task-system config-overrides))) - -(defn onyx-system - "Creates a base-system and assocs in polismath onyx worker related components." - [config-overrides] - :TODO) - -(defn simulator-system - "Creates a base-system and assocs in a simulation engine." - [config-overrides] - :TODO) + (poller-system config-overrides))) (defn create-and-run-system! [system config] diff --git a/math/src/polismath/tasks.clj b/math/src/polismath/tasks.clj index 97f9fee01a..f3302a81c5 100644 --- a/math/src/polismath/tasks.clj +++ b/math/src/polismath/tasks.clj @@ -2,15 +2,13 @@ (ns polismath.tasks (:require - [clojure.core.async :as async :refer [go go-loop >! !! ! go]] + [com.stuartsierra.component :as component] + [polismath.components.postgres :as postgres] + [polismath.conv-man :as conv-man] + [polismath.darwin.core :as darwin] + [polismath.darwin.export :as export] + [taoensso.timbre :as log])) ;; This is where we listen/poll for and dispatch tasks posted in the :worker_tasks table. ;; The polling code should eventually be unified with the other vote polling code, but for now there are enough subtle differences that we'll bite the bullet on that till later. @@ -34,9 +32,9 @@ (defmethod dispatch-task! :generate_export_data - [{:as poller :keys [darwin]} task-record] + [{:keys [darwin]} task-record] (log/debug "Dispatching generate_export_data for" task-record) - (let [params (assoc (:task_data task-record) :task_bucket (:task_bucket task-record)) + (let [_params (assoc (:task_data task-record) :task_bucket (:task_bucket task-record)) ;; Need to think about security implications more thoroughly before we can really include this from ;; requests triggered by server (needs thorough audit to make sure user can't trigger this via web ;; api request) @@ -52,7 +50,7 @@ (defmethod dispatch-task! :update_math - [{:as poller :keys [darwin conversation-manager]} task-record] + [{:keys [conversation-manager]} task-record] (log/debug "Dispatching update_math task for:" task-record) (async/thread (conv-man/queue-message-batch! conversation-manager :votes (-> task-record :task_data :zid) []))) diff --git a/math/src/polismath/util/pretty_printers.clj b/math/src/polismath/util/pretty_printers.clj deleted file mode 100644 index 562a477f85..0000000000 --- a/math/src/polismath/util/pretty_printers.clj +++ /dev/null @@ -1,29 +0,0 @@ -;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . - -(ns polismath.util.pretty-printers - ;; Shoud just switch to fipp - (:require [clojure.pprint :refer :all])) - - -(defn str-repeat [s n] - (apply str (repeat n s))) - - -(defn prindent [base-indent main-indent & other-strings] - (print (str-repeat " " (* 2 (+ base-indent main-indent)))) - (apply println other-strings)) - - -(defn print-repness [repness & {:keys [indent] :or {indent 0}}] - (doseq [[gid comments] repness] - (prindent indent 0 gid) - (doseq [c comments] - (prindent indent 2 c)))) - - -(defn wide-pp [x & {:keys [width] :or {width 400}}] - (binding [*print-miser-width* width - *print-right-margin* width] - (pprint x))) - - diff --git a/math/src/polismath/utils.clj b/math/src/polismath/utils.clj index 28c0792595..6defabfb67 100644 --- a/math/src/polismath/utils.clj +++ b/math/src/polismath/utils.clj @@ -1,20 +1,17 @@ ;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . (ns polismath.utils - (:use clojure.core.matrix) - (:require [clojure.core.matrix :as matrix] - [clojure.math.numeric-tower :as math] - [clojure.core.matrix :as mat] - [clojure.tools.trace :as tr])) - + (:require + [clojure.core.matrix :as matrix] + [clojure.math.numeric-tower :as math])) (matrix/set-current-implementation :vectorz) (defn xor [a b] (and - (or a b) - (not (and a b)))) + (or a b) + (not (and a b)))) (defn round-to @@ -26,12 +23,6 @@ (/ tens)))) -(defn gets - "Like get, but gives a coll mapped from all the keys" - [m ks & [not-found]] - (mapv #(get m % not-found) ks)) - - (defn exit [status msg] (println msg) (System/exit status)) @@ -39,44 +30,19 @@ (defn agree? [n] (and - (not (nil? n)) - (< n 0))) + (not (nil? n)) + (< n 0))) (defn disagree? [n] (and - (not (nil? n)) - (> n 0))) - - -(defmacro time2 - [tag & expr] - `(let [start# (. System (nanoTime)) - ret# ~@expr] - (println (str (System/currentTimeMillis) " " ~tag " " (/ (double (- (. System (nanoTime)) start#)) 1000000.0) " msecs")) - ret#)) - - -(defmacro f?>> - "Modified 'penguin' operator from plumbing.core, where do-it? is a function of the threaded value - instead of a static value. E.g.: (->> nums (f?>> #(even? (count %)) (map inc)))" - [do-it? & args] - `(if (~do-it? ~(last args)) - (->> ~(last args) ~@(butlast args)) - ~(last args))) - + (not (nil? n)) + (> n 0))) -(defmacro f?> - "Modified 'penguin' operator from plumbing.core, where do-it? is a function of the threaded value - instead of a static value. E.g.: (-> n inc (f?> even? (* 2)))" - [arg do-it? & rest] - `(if (~do-it? ~arg) - (-> ~arg ~@rest) - ~arg)) - -(defn zip [& xss] +(defn zip "Like haskell or python zip" + [& xss] ;;should we redo this like the with-indices below, using a map? (if (> (count xss) 1) (partition (count xss) (apply interleave xss)) @@ -90,8 +56,8 @@ (for [i (range (count col))] (f (get col i) (concat - (subvec col 0 i) - (subvec col (inc i)))))) + (subvec col 0 i) + (subvec col (inc i)))))) (defn mapv-rest @@ -100,37 +66,23 @@ (vec (map-rest f col))) -;; XX This should be an env variable -(let [greedy? true] - (defn greedy [iter] - (if greedy? - (into [] iter) - iter))) - - (defn greedy-false [iter] iter) -(defn ^long typed-indexof [^java.util.List coll item] +(defn typed-indexof ^long [^java.util.List coll item] (.indexOf coll item)) -(defmacro endlessly [interval & forms] - `(doseq [~'x (range)] - ~@forms - (Thread/sleep ~interval))) - - (defn with-indices [coll] (map #(vector %1 %2) (range) coll)) -(defn filter-by-index [coll idxs] +(defn filter-by-index [coll idxs] (greedy-false (let [idx-set (set idxs)] (->> (with-indices coll) - (filter #(idx-set (first %))) - (map second))))) + (filter #(idx-set (first %))) + (map second))))) @@ -142,22 +94,13 @@ (defn hash-map-subset - "Create a new map which is given by subsetting to the given keys (ks)" - [m ks] - (let [ks (set ks)] - (into {} - (filter - (fn [[k v]] (ks k)) - m)))) - - -(defn clst-trace - ([clsts] (clst-trace "" clsts)) - ([k clsts] - (println "TRACE" k ":") - (doseq [c clsts] - (println " " c)) - clsts)) + "Create a new map which is given by subsetting to the given keys (ks)" + [m ks] + (let [ks (set ks)] + (into {} + (filter + (fn [[k _v]] (ks k)) + m)))) (comment @@ -165,8 +108,7 @@ (defn load-dep [dep] (add-dependencies :coordinates [dep] :repositories (merge cemerick.pomegranate.aether/maven-central {"clojars" "http://clojars.org/repo"}))) - (load-dep '[clj-time "0.10.0"]) - (load-dep '[clj-excel "0.0.1"])) + (load-dep '[clj-time "0.10.0"])) :ok diff --git a/math/test/conv_man_tests.clj b/math/test/conv_man_tests.clj index 772638a303..bb6cf0b1ee 100644 --- a/math/test/conv_man_tests.clj +++ b/math/test/conv_man_tests.clj @@ -4,9 +4,6 @@ (:use polismath.utils test-helpers) (:require [clojure.test :refer :all] - ;[clojure.test.check :as tc] - ;[clojure.test.check.generators :as gen] - ;[clojure.test.check.properties :as prop] [clojure.core.async :as async] [clojure.tools.logging :as log] [polismath.math.conversation :as conv] @@ -17,10 +14,6 @@ -;(gen/sample gen/->Generator - -;(let [random-matrix (gen/vector (partial gen/vector (ge - ;; Should make this generative (deftest integration-test (testing "of some votes" @@ -28,8 +21,7 @@ {:vote 1 :pid 1 :tid 1 :zid 0 :created 9003} {:vote -1 :pid 2 :tid 0 :zid 0 :created 9005} {:vote -1 :pid 1 :tid 2 :zid 0 :created 9009}] - config {:math-env :test - :poller {:initial-polling-timestamp (System/currentTimeMillis)}}] + config {:poller {:initial-polling-timestamp (System/currentTimeMillis)}}] (testing "in a new conv" (let [system (system/create-and-run-base-system! config) conv-man (:conversation-manager system) diff --git a/math/tmp/.touchfile b/math/tmp/.touchfile deleted file mode 100644 index e69de29bb2..0000000000 From 47f8bddd9d33989bdc4c6300feacbfbda21c620e Mon Sep 17 00:00:00 2001 From: Bennie Rosas Date: Sat, 25 Oct 2025 05:02:21 -0500 Subject: [PATCH 5/7] remove MATH_ENV and WEBSERVER_* env vars --- docker-compose.test.yml | 3 --- docker-compose.yml | 3 --- docs/configuration.md | 3 --- example.env | 5 ----- test.env | 3 --- 5 files changed, 17 deletions(-) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 36c6f9542a..da6eba6e71 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -64,9 +64,6 @@ services: - DATABASE_URL=${DATABASE_URL} - DD_ENABLED=false - LOGGING_LEVEL=${MATH_LOG_LEVEL:-warn} - - MATH_ENV=${MATH_ENV:-dev} - - WEBSERVER_USERNAME=${WEBSERVER_USERNAME} - - WEBSERVER_PASS=${WEBSERVER_PASS} networks: - polis-test restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index 7e2712706f..956109415d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -98,9 +98,6 @@ services: environment: - DATABASE_URL=${DATABASE_URL} - LOGGING_LEVEL=${MATH_LOG_LEVEL:-warn} - - MATH_ENV=${MATH_ENV:-prod} - - WEBSERVER_USERNAME=${WEBSERVER_USERNAME} - - WEBSERVER_PASS=${WEBSERVER_PASS} # Database connection pool configuration for math service - DATABASE_POOL_SIZE=${MATH_DATABASE_POOL_SIZE:-10} - DD_AGENT_HOST=datadog-agent diff --git a/docs/configuration.md b/docs/configuration.md index 7be1da32ae..8523cd1a3e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -55,7 +55,6 @@ If you are deploying to a custom domain (not `pol.is`) then you need to update b - **`ADMIN_UIDS`** an array of user UIDs for site admins. These users will have moderator capabilities on all conversations hosted on the site. - **`EMAIL_TRANSPORT_TYPES`** comma-separated list of email services to use (see [Email Transports](#email-transports) below) - **`GIT_HASH`** Set programmatically using `git rev-parse HEAD` (e.g. in `Makefile`) to tag docker container versions and other release assets. Can be left blank. -- **`MATH_ENV`** Set to `prod` (default), `preprod`, `dev` or arbitrary feature flag. In cases where a single database is used for multiple environments, this value is used by the API service to request the correct data. (Using a single DB for multiple environments in this fashion is no longer recommended, and so the default value of `prod` is recommended.) - **`MATH_LOG_LEVEL`** Used by the math service to determine how much logging to output. Reasonable values are `debug`, `info`, `warn`, and `error`. Defaults to `warn`. - **`SERVER_ENV_FILE`** The name of an environment file to be passed into the API Server container by docker compose. Defaults to `.env` if left blank. Used especially for building a `test` version of the project for end-to-end testing. - **`SERVER_LOG_LEVEL`** Used by Winston.js in the API server to determine how much logging to output. Reasonable values are `debug`, `info`, `warn`, and `error`. Defaults to `warn`. @@ -159,8 +158,6 @@ If you are deploying to a custom domain (not `pol.is`) then you need to update b ### Deprecated - **`ENCRYPTION_PASSWORD_00001`** (deprecated) a password used to encrypt and decrypt participants' IP addresses. Can be left blank. -- **`WEBSERVER_PASS`** (deprecated) basic auth setting for certain requests sent between math and api services. -- **`WEBSERVER_USERNAME`** (deprecated) basic auth setting for certain requests sent between math and api services. ## Enabling Comment Translation diff --git a/example.env b/example.env index f33c7ca42a..20a9956640 100644 --- a/example.env +++ b/example.env @@ -6,8 +6,6 @@ ADMIN_UIDS= # Optionally set this manually or use `git rev-parse HEAD`. It can be useful for debugging. GIT_HASH= -# Options: prod, preprod, dev: -MATH_ENV=dev # Options: debug, info, warn, error, fatal. Default is warn. MATH_LOG_LEVEL= # Optionally give the server container a distinct env_file. Useful for CI tests. @@ -215,6 +213,3 @@ TOPICAL_COMMENT_RATIO=0.6 ###### DEPRECATED ###### # (Deprecated) Used internally by Node.Crypto to encrypt/decrypt IP addresses. ENCRYPTION_PASSWORD_00001= -# (Deprecated) Basic Auth settings for certain requests between math and api services. -WEBSERVER_PASS=ws-pass -WEBSERVER_USERNAME=ws-user diff --git a/test.env b/test.env index d7d4dbf6fd..83d8e4f509 100644 --- a/test.env +++ b/test.env @@ -23,10 +23,7 @@ POSTGRES_PORT=5432 POSTGRES_USER=postgres DATABASE_URL=postgres://postgres:PdwPNS2mDN73Vfbc@postgres:5432/polis-test -MATH_ENV=dev MATH_LOG_LEVEL=warn -WEBSERVER_PASS=ws-pass -WEBSERVER_USERNAME=ws-user # Set to true and provide valid GOOGLE_APPLICATION_CREDENTIALS to test translation SHOULD_USE_TRANSLATION_API=false From d1bda248835ea93c583f66fea417aaa4a549c3be Mon Sep 17 00:00:00 2001 From: Bennie Rosas Date: Sat, 25 Oct 2025 05:20:22 -0500 Subject: [PATCH 6/7] math fix --- math/src/polismath/math/conversation.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/math/src/polismath/math/conversation.clj b/math/src/polismath/math/conversation.clj index 031d399458..03328cacde 100644 --- a/math/src/polismath/math/conversation.clj +++ b/math/src/polismath/math/conversation.clj @@ -611,7 +611,7 @@ extremities (into {} (map vector tids (:comment-extremity pca)))] (plmb/map-from-keys (fn [tid] - (let [{:keys [A P]} + (let [{:keys [A P S]} ;; reduce over votes per group, already aggregated (reduce (fn [votes [_gid data]] From cc0238c0b6d6f4930d4dcb7577eab0018f444bdd Mon Sep 17 00:00:00 2001 From: Bennie Rosas Date: Sat, 25 Oct 2025 05:28:36 -0500 Subject: [PATCH 7/7] remove tmp file --- delphi/schema.sql | 3512 --------------------------------------------- 1 file changed, 3512 deletions(-) delete mode 100644 delphi/schema.sql diff --git a/delphi/schema.sql b/delphi/schema.sql deleted file mode 100644 index 0d1d2d018e..0000000000 --- a/delphi/schema.sql +++ /dev/null @@ -1,3512 +0,0 @@ --- --- PostgreSQL database dump --- - -\restrict yi3q8cVeyPYXP7xpzDDwpdv2ts6MbtdVFjxHM5kY0sd3iTqn6dEoZ2y0ASHXmgp - --- Dumped from database version 17.6 --- Dumped by pg_dump version 18.0 - -SET statement_timeout = 0; -SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; -SET transaction_timeout = 0; -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = on; -SELECT pg_catalog.set_config('search_path', '', false); -SET check_function_bodies = false; -SET xmloption = content; -SET client_min_messages = warning; -SET row_security = off; - --- --- Name: pg_stat_statements; Type: EXTENSION; Schema: -; Owner: - --- - -CREATE EXTENSION IF NOT EXISTS pg_stat_statements WITH SCHEMA public; - - --- --- Name: EXTENSION pg_stat_statements; Type: COMMENT; Schema: -; Owner: - --- - -COMMENT ON EXTENSION pg_stat_statements IS 'track execution statistics of all SQL statements executed'; - - --- --- Name: tablefunc; Type: EXTENSION; Schema: -; Owner: - --- - -CREATE EXTENSION IF NOT EXISTS tablefunc WITH SCHEMA public; - - --- --- Name: EXTENSION tablefunc; Type: COMMENT; Schema: -; Owner: - --- - -COMMENT ON EXTENSION tablefunc IS 'functions that manipulate whole tables, including crosstab'; - - --- --- Name: animal_grp; Type: TYPE; Schema: public; Owner: - --- - -CREATE TYPE public.animal_grp AS ENUM ( - 'fish', - 'mammal', - 'bird' -); - - --- --- Name: animals_id_auto(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.animals_id_auto() RETURNS trigger - LANGUAGE plpgsql STRICT - AS $$ -DECLARE - _rel_id constant int := 'animals'::regclass::int; - _grp_id int; -BEGIN - _grp_id = array_length(enum_range(NULL, NEW.grp), 1); - - -- Obtain an advisory lock on this table/group. - PERFORM pg_advisory_lock(_rel_id, _grp_id); - - SELECT COALESCE(MAX(id) + 1, 1) - INTO NEW.id - FROM animals - WHERE grp = NEW.grp; - - RETURN NEW; -END; -$$; - - --- --- Name: get_times_for_most_recent_visible_comments(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.get_times_for_most_recent_visible_comments() RETURNS TABLE(zid integer, modified bigint) - LANGUAGE sql - AS $$ - select zid, max(modified) from (select comments.*, conversations.strict_moderation from comments left join conversations on comments.zid = conversations.zid) as c where c.mod >= (CASE WHEN c.strict_moderation=TRUE then 1 else 0 END) group by c.zid order by c.zid; -$$; - - --- --- Name: get_visible_comments(integer); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.get_visible_comments(the_zid integer) RETURNS TABLE(tid integer, mod integer, strict_moderation boolean) - LANGUAGE sql - AS $$ - select comments.tid, comments.mod, conversations.strict_moderation from comments left join conversations on comments.zid = conversations.zid where active = true and mod >= (CASE WHEN strict_moderation=TRUE then 1 else 0 END) and comments.zid = the_zid; -$$; - - --- --- Name: now_as_millis(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.now_as_millis() RETURNS bigint - LANGUAGE plpgsql - AS $$ - DECLARE - temp TIMESTAMP := now(); - BEGIN - -- NOTE: milliseconds includes the seconds, so subtracting seconds from milliseconds - -- SEE: http://www.postgresql.org/docs/8.4/static/functions-datetime.html - RETURN 1000*FLOOR(EXTRACT(EPOCH FROM temp)) + FLOOR(EXTRACT(MILLISECONDS FROM temp)) - 1000*FLOOR(EXTRACT(SECOND FROM temp)); - END; -$$; - - --- --- Name: oid_auto(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.oid_auto() RETURNS trigger - LANGUAGE plpgsql STRICT - AS $$ -DECLARE - _magic_id constant int := 873791984; -- This is a magic key used for locking conversation row-sets within the opinions table. TODO keep track of these - _opinion_id int; -BEGIN - _opinion_id = NEW.oid; - - -- Obtain an advisory lock on the opinions table, limited to this conversation - PERFORM pg_advisory_lock(_magic_id, _opinion_id); - - SELECT COALESCE(MAX(oid) + 1, 1) - INTO NEW.oid - FROM opinions - WHERE oid = NEW.oid; - - RETURN NEW; -END; -$$; - - --- --- Name: oid_auto_unlock(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.oid_auto_unlock() RETURNS trigger - LANGUAGE plpgsql STRICT - AS $$ -DECLARE - _magic_id constant int := 873791984; - _opinion_id int; -BEGIN - _opinion_id = NEW.oid; - - -- Release the lock. - PERFORM pg_advisory_unlock(_magic_id, _opinion_id); - - RETURN NEW; -END; -$$; - - --- --- Name: pid_auto(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.pid_auto() RETURNS trigger - LANGUAGE plpgsql STRICT - AS $$ -DECLARE - _magic_id constant int := 873791983; -- This is a magic key used for locking conversation row-sets within the participants table. TODO keep track of these - _conversation_id int; -BEGIN - _conversation_id = NEW.zid; - - -- Obtain an advisory lock on the participants table, limited to this conversation - PERFORM pg_advisory_lock(_magic_id, _conversation_id); - - SELECT COALESCE(MAX(pid) + 1, 0) -- Start with comment id of 0 - INTO NEW.pid - FROM participants - WHERE zid = NEW.zid; - - -- Duplicate participant_count to the conversations table to speed up conversationsView queries. - UPDATE conversations - SET participant_count = NEW.pid + 1 - WHERE zid = NEW.zid; - - RETURN NEW; -END; -$$; - - --- --- Name: pid_auto_unlock(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.pid_auto_unlock() RETURNS trigger - LANGUAGE plpgsql STRICT - AS $$ -DECLARE - _magic_id constant int := 873791983; - _conversation_id int; -BEGIN - _conversation_id = NEW.zid; - - -- Release the lock. - PERFORM pg_advisory_unlock(_magic_id, _conversation_id); - - RETURN NEW; -END; -$$; - - --- --- Name: ptpt_id_auto(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.ptpt_id_auto() RETURNS trigger - LANGUAGE plpgsql STRICT - AS $$ -DECLARE - _rel_id constant int := 873791983; -- This is a magic key used for locking conversation row-sets within the participants table. TODO keep track of these - _grp_id int; -BEGIN - _grp_id = NEW.conv_id; - - -- Obtain an advisory lock on the participants table, limited to this conversation - PERFORM pg_advisory_lock(_rel_id, _grp_id); - - SELECT COALESCE(MAX(ptpt_id) + 1, 1) - INTO NEW.ptpt_id - FROM participants - WHERE conv_id = NEW.conv_id; - - RETURN NEW; -END; -$$; - - --- --- Name: ptpt_id_auto_unlock(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.ptpt_id_auto_unlock() RETURNS trigger - LANGUAGE plpgsql STRICT - AS $$ -DECLARE - _rel_id constant int := 873791983; - _grp_id int; -BEGIN - _grp_id = NEW.conv_id; - - -- Release the lock. - PERFORM pg_advisory_unlock(_rel_id, _grp_id); - - RETURN NEW; -END; -$$; - - --- --- Name: random_polis_site_id(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.random_polis_site_id() RETURNS text - LANGUAGE sql - AS $$ -SELECT 'polis_site_id_' || random_string(18); -$$; - - --- --- Name: random_polis_site_id(integer); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.random_polis_site_id(integer) RETURNS text - LANGUAGE sql - AS $$ -SELECT 'polis_site_id_' || random_string(18); -$$; - - --- --- Name: random_string(integer); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.random_string(integer) RETURNS text - LANGUAGE sql - AS $_$ -SELECT array_to_string( - ARRAY ( - SELECT substring( - '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' - FROM (ceil(random()*62))::int FOR 1 - ) - FROM generate_series(1, $1) - ), - '' -) -$_$; - - --- --- Name: tid_auto(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.tid_auto() RETURNS trigger - LANGUAGE plpgsql STRICT - AS $$ -DECLARE - _magic_id constant int := 873791984; -- This is a magic key used for locking conversation row-sets within the comments table. TODO keep track of these - _conversation_id int; -BEGIN - _conversation_id = NEW.zid; - - -- Obtain an advisory lock on the comments table, limited to this conversation - PERFORM pg_advisory_lock(_magic_id, _conversation_id); - - SELECT COALESCE(MAX(tid) + 1, 0) -- Start with comment id of 0 - INTO NEW.tid - FROM comments - WHERE zid = NEW.zid; - - RETURN NEW; -END; -$$; - - --- --- Name: tid_auto_unlock(); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.tid_auto_unlock() RETURNS trigger - LANGUAGE plpgsql STRICT - AS $$ -DECLARE - _magic_id constant int := 873791984; - _conversation_id int; -BEGIN - _conversation_id = NEW.zid; - - -- Release the lock. - PERFORM pg_advisory_unlock(_magic_id, _conversation_id); - - RETURN NEW; -END; -$$; - - --- --- Name: to_millis(timestamp with time zone); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.to_millis(t timestamp with time zone) RETURNS bigint - LANGUAGE plpgsql - AS $$ - BEGIN - RETURN 1000*FLOOR(EXTRACT(EPOCH FROM t)) + FLOOR(EXTRACT(MILLISECONDS FROM t)) - 1000*FLOOR(EXTRACT(SECOND FROM t)); - END; -$$; - - --- --- Name: to_zid(text); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.to_zid(associated_zinvite text) RETURNS integer - LANGUAGE plpgsql - AS $$ - BEGIN - RETURN (select zid from zinvites where zinvite = associated_zinvite); - END; -$$; - - --- --- Name: to_zinvite(integer); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.to_zinvite(associated_zid integer) RETURNS text - LANGUAGE plpgsql - AS $$ - BEGIN - RETURN (select zinvite from zinvites where zid = associated_zid); - END; -$$; - - -SET default_tablespace = ''; - -SET default_table_access_method = heap; - --- --- Name: votes; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.votes ( - zid integer NOT NULL, - pid integer NOT NULL, - tid integer NOT NULL, - vote smallint, - created bigint DEFAULT public.now_as_millis(), - weight_x_32767 smallint DEFAULT 0, - high_priority boolean DEFAULT false NOT NULL -); - - --- --- Name: votes_foo(integer); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.votes_foo(integer) RETURNS SETOF public.votes - LANGUAGE sql - AS $_$ - select * from votes where zid = $1; -$_$; - - --- --- Name: votes_lastest_unique(integer); Type: FUNCTION; Schema: public; Owner: - --- - -CREATE FUNCTION public.votes_lastest_unique(integer) RETURNS SETOF public.votes - LANGUAGE sql - AS $_$ - WITH m AS (SELECT zid, pid, tid, MAX(created) AS created FROM votes WHERE zid = $1 GROUP BY zid, pid, tid) SELECT v.* FROM m LEFT JOIN votes v ON m.zid = v.zid AND m.pid = v.pid AND m.tid = v.tid WHERE m.created = v.created; -$_$; - - --- --- Name: apikeysndvweifu; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.apikeysndvweifu ( - uid integer NOT NULL, - apikey character varying(32) NOT NULL, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: auth_tokens; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.auth_tokens ( - token character varying(32), - uid integer, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: beta; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.beta ( - name character varying(999), - email character varying(200), - organization character varying(200), - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: comment_translations; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.comment_translations ( - zid integer NOT NULL, - tid integer NOT NULL, - src integer NOT NULL, - txt character varying(9999) NOT NULL, - lang character varying(10) NOT NULL, - created bigint DEFAULT public.now_as_millis(), - modified bigint DEFAULT public.now_as_millis() -); - - --- --- Name: comments; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.comments ( - tid integer NOT NULL, - zid integer NOT NULL, - pid integer NOT NULL, - txt character varying(1000) NOT NULL, - created bigint DEFAULT public.now_as_millis(), - velocity real DEFAULT 1, - mod integer DEFAULT 0 NOT NULL, - active boolean DEFAULT true NOT NULL, - modified bigint DEFAULT public.now_as_millis(), - uid integer DEFAULT 0 NOT NULL, - tweet_id bigint, - quote_src_url character varying(1000), - anon boolean DEFAULT false NOT NULL, - is_seed boolean DEFAULT false NOT NULL, - curation smallint DEFAULT 0 NOT NULL, - is_meta boolean DEFAULT false NOT NULL, - lang_confidence real, - lang character varying(10) -); - - --- --- Name: contexts; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.contexts ( - context_id integer NOT NULL, - name character varying(300), - creator integer, - is_public boolean DEFAULT false, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: contexts_context_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.contexts_context_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: contexts_context_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.contexts_context_id_seq OWNED BY public.contexts.context_id; - - --- --- Name: contributor_agreement_signatures; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.contributor_agreement_signatures ( - uid integer, - name character varying(746) NOT NULL, - github_id character varying(256), - email character varying(256) NOT NULL, - agreement_version integer NOT NULL, - created bigint DEFAULT public.now_as_millis(), - company_name character varying(746) -); - - --- --- Name: conversation_invite_codes; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.conversation_invite_codes ( - zid integer NOT NULL, - code character varying(300) NOT NULL, - created timestamp with time zone DEFAULT now() -); - - --- --- Name: conversation_subscriptions; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.conversation_subscriptions ( - zid integer NOT NULL, - uid integer NOT NULL, - time_last_notified bigint DEFAULT 0, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: conversation_translations; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.conversation_translations ( - zid integer NOT NULL, - src integer NOT NULL, - topic character varying(9999) NOT NULL, - description character varying(9999) NOT NULL, - lang character varying(10) NOT NULL, - created bigint DEFAULT public.now_as_millis(), - modified bigint DEFAULT public.now_as_millis() -); - - --- --- Name: conversations; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.conversations ( - zid integer NOT NULL, - topic character varying(1000), - description character varying(50000), - is_anon boolean DEFAULT true, - is_active boolean DEFAULT false, - is_draft boolean DEFAULT false, - is_public boolean DEFAULT true, - email_domain character varying(200), - owner integer, - participant_count integer DEFAULT 0, - created bigint DEFAULT public.now_as_millis(), - strict_moderation boolean DEFAULT false, - profanity_filter boolean DEFAULT true, - spam_filter boolean DEFAULT true, - context character varying(1000), - modified bigint DEFAULT public.now_as_millis(), - owner_sees_participation_stats boolean DEFAULT false, - course_id integer, - link_url character varying(9999), - upvotes integer DEFAULT 1 NOT NULL, - parent_url character varying(9999), - vis_type integer DEFAULT 0 NOT NULL, - write_type integer DEFAULT 1 NOT NULL, - bgcolor character varying(20), - help_type integer DEFAULT 1 NOT NULL, - socialbtn_type integer DEFAULT 0 NOT NULL, - style_btn character varying(500), - auth_needed_to_vote boolean DEFAULT false, - auth_needed_to_write boolean DEFAULT true, - auth_opt_fb boolean DEFAULT true, - auth_opt_tw boolean DEFAULT true, - auth_opt_allow_3rdparty boolean DEFAULT true, - help_bgcolor character varying(20), - help_color character varying(20), - is_data_open boolean DEFAULT false, - is_curated boolean DEFAULT false, - dataset_explanation character varying(50000), - write_hint_type integer DEFAULT 1 NOT NULL, - subscribe_type integer DEFAULT 1 NOT NULL, - org_id integer, - need_suzinvite boolean DEFAULT false, - use_xid_whitelist boolean DEFAULT false, - prioritize_seed boolean DEFAULT false, - importance_enabled boolean DEFAULT false NOT NULL, - treevite_enabled boolean DEFAULT false -); - - --- --- Name: COLUMN conversations.treevite_enabled; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.conversations.treevite_enabled IS 'Enable wave-based invite (Treevite) for this conversation'; - - --- --- Name: conversations_zid_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.conversations_zid_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: conversations_zid_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.conversations_zid_seq OWNED BY public.conversations.zid; - - --- --- Name: courses; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.courses ( - course_id integer NOT NULL, - topic character varying(1000), - description character varying(1000), - owner integer, - created bigint DEFAULT public.now_as_millis(), - course_invite character varying(32) -); - - --- --- Name: courses_course_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.courses_course_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: courses_course_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.courses_course_id_seq OWNED BY public.courses.course_id; - - --- --- Name: crowd_mod; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.crowd_mod ( - zid integer NOT NULL, - pid integer NOT NULL, - tid integer NOT NULL, - created bigint DEFAULT public.now_as_millis(), - as_important boolean, - as_factual boolean, - as_feeling boolean, - as_notmyfeeling boolean, - as_notgoodidea boolean, - as_notfact boolean, - as_unsure boolean, - as_spam boolean, - as_abusive boolean, - as_offtopic boolean -); - - --- --- Name: demographic_data; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.demographic_data ( - uid integer, - fb_gender integer, - ms_birth_year_estimate_fb integer, - ms_gender_estimate_fb integer, - fb_timestamp bigint DEFAULT public.now_as_millis(), - ms_fb_timestamp bigint DEFAULT public.now_as_millis(), - ms_response character varying(9999), - gender_guess integer, - birth_year_guess integer -); - - --- --- Name: einvites; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.einvites ( - einvite character varying(100) NOT NULL, - email character varying(999), - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: email_validations; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.email_validations ( - email character varying(999), - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: error_reports; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.error_reports ( - uid integer NOT NULL, - zid integer NOT NULL, - error_code character varying(99), - data character varying(9999), - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: event_ptpt_no_more_comments; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.event_ptpt_no_more_comments ( - zid integer NOT NULL, - pid integer NOT NULL, - votes_placed smallint NOT NULL, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: facebook_friends; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.facebook_friends ( - uid integer NOT NULL, - friend integer NOT NULL -); - - --- --- Name: facebook_users; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.facebook_users ( - uid integer NOT NULL, - fb_user_id text, - fb_public_profile text, - fb_login_status text, - fb_auth_response text, - fb_access_token text, - fb_granted_scopes text, - response text, - fb_friends_response text, - created bigint DEFAULT public.now_as_millis(), - location character varying(9999), - fb_location_id character varying(100), - fb_name character varying(9999), - fb_link character varying(9999), - modified bigint DEFAULT public.now_as_millis() -); - - --- --- Name: foobar; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.foobar - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: inviters; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.inviters ( - inviter_uid integer, - invited_email character varying(999), - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: jianiuevyew; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.jianiuevyew ( - uid integer NOT NULL, - pwhash character varying(128) NOT NULL -); - - --- --- Name: math_bidtopid; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.math_bidtopid ( - zid integer NOT NULL, - math_env character varying(999) NOT NULL, - data json NOT NULL, - modified bigint DEFAULT public.now_as_millis(), - math_tick bigint DEFAULT '-1'::integer NOT NULL -); - - --- --- Name: math_cache; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.math_cache ( - zid integer NOT NULL, - math_env character varying(999) NOT NULL, - data json NOT NULL, - modified bigint DEFAULT public.now_as_millis() -); - - --- --- Name: math_exportstatus; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.math_exportstatus ( - zid integer NOT NULL, - math_env character varying(999) NOT NULL, - filename character varying(9999) NOT NULL, - data json NOT NULL, - modified bigint DEFAULT public.now_as_millis() -); - - --- --- Name: math_main; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.math_main ( - zid integer NOT NULL, - math_env character varying(999) NOT NULL, - data json NOT NULL, - last_vote_timestamp bigint NOT NULL, - modified bigint DEFAULT public.now_as_millis(), - math_tick bigint DEFAULT '-1'::integer NOT NULL, - caching_tick bigint DEFAULT 0 NOT NULL -); - - --- --- Name: math_profile; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.math_profile ( - zid integer NOT NULL, - math_env character varying(999) NOT NULL, - data json NOT NULL, - modified bigint DEFAULT public.now_as_millis() -); - - --- --- Name: math_ptptstats; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.math_ptptstats ( - zid integer NOT NULL, - math_env character varying(999) NOT NULL, - data json NOT NULL, - modified bigint DEFAULT public.now_as_millis(), - math_tick bigint DEFAULT '-1'::integer NOT NULL -); - - --- --- Name: math_report_correlationmatrix; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.math_report_correlationmatrix ( - rid bigint NOT NULL, - math_env character varying(999) NOT NULL, - data jsonb, - modified bigint DEFAULT public.now_as_millis(), - math_tick bigint DEFAULT '-1'::integer NOT NULL -); - - --- --- Name: math_results_dev01; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.math_results_dev01 ( - zid integer NOT NULL, - last_timestamp bigint NOT NULL, - data text -); - - --- --- Name: math_ticks; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.math_ticks ( - zid integer, - math_tick bigint DEFAULT 0 NOT NULL, - math_env character varying(999) NOT NULL, - modified bigint DEFAULT public.now_as_millis() NOT NULL -); - - --- --- Name: metrics; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.metrics ( - uid integer, - type integer NOT NULL, - dur integer, - created bigint DEFAULT public.now_as_millis(), - hashedpc integer -); - - --- --- Name: minvites; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.minvites ( - zid integer NOT NULL, - minvite character varying(300) NOT NULL, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: moderators; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.moderators ( - zid integer NOT NULL, - uid integer, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: notification_tasks; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.notification_tasks ( - zid integer NOT NULL, - modified bigint DEFAULT public.now_as_millis() -); - - --- --- Name: nyt_users; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.nyt_users ( - uid integer NOT NULL, - nyt_user_id bigint NOT NULL, - nyt_name character varying(9999), - nyt_img character varying(9999), - modified bigint DEFAULT public.now_as_millis() NOT NULL, - created bigint DEFAULT public.now_as_millis() NOT NULL -); - - --- --- Name: oidc_user_mappings; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.oidc_user_mappings ( - oidc_sub character varying(255) NOT NULL, - uid integer NOT NULL, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: COLUMN oidc_user_mappings.oidc_sub; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.oidc_user_mappings.oidc_sub IS 'OIDC subject (sub) claim from JWT'; - - --- --- Name: COLUMN oidc_user_mappings.uid; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.oidc_user_mappings.uid IS 'Local Polis user ID'; - - --- --- Name: COLUMN oidc_user_mappings.created; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.oidc_user_mappings.created IS 'Timestamp when mapping was created'; - - --- --- Name: oinvites; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.oinvites ( - oinvite character varying(300) NOT NULL, - note character varying(999), - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: page_ids; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.page_ids ( - site_id character varying(100) NOT NULL, - page_id character varying(100) NOT NULL, - zid integer NOT NULL -); - - --- --- Name: participant_locations; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.participant_locations ( - zid integer NOT NULL, - uid integer NOT NULL, - pid integer NOT NULL, - lat double precision NOT NULL, - lng double precision NOT NULL, - created bigint DEFAULT public.now_as_millis(), - source integer NOT NULL -); - - --- --- Name: participant_metadata_answers; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.participant_metadata_answers ( - pmaid integer NOT NULL, - pmqid integer, - zid integer, - value character varying(999), - alive boolean DEFAULT true, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: participant_metadata_answers_pmaid_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.participant_metadata_answers_pmaid_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: participant_metadata_answers_pmaid_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.participant_metadata_answers_pmaid_seq OWNED BY public.participant_metadata_answers.pmaid; - - --- --- Name: participant_metadata_choices; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.participant_metadata_choices ( - zid integer, - pid integer, - pmqid integer, - pmaid integer, - alive boolean DEFAULT true, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: participant_metadata_questions; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.participant_metadata_questions ( - pmqid integer NOT NULL, - zid integer, - key character varying(999), - alive boolean DEFAULT true, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: participant_metadata_questions_pmqid_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.participant_metadata_questions_pmqid_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: participant_metadata_questions_pmqid_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.participant_metadata_questions_pmqid_seq OWNED BY public.participant_metadata_questions.pmqid; - - --- --- Name: participants; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.participants ( - pid integer NOT NULL, - uid integer NOT NULL, - zid integer NOT NULL, - created bigint DEFAULT public.now_as_millis(), - vote_count integer DEFAULT 0 NOT NULL, - last_interaction bigint DEFAULT 0 NOT NULL, - subscribed integer DEFAULT 0 NOT NULL, - last_notified bigint DEFAULT 0, - mod integer DEFAULT 0 NOT NULL, - nsli smallint DEFAULT 0 NOT NULL -); - - --- --- Name: participants_extended; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.participants_extended ( - uid integer NOT NULL, - zid integer NOT NULL, - referrer character varying(9999), - parent_url character varying(9999), - created bigint DEFAULT public.now_as_millis(), - permanent_cookie character varying(32), - origin character varying(9999), - encrypted_ip_address character varying(9999), - encrypted_x_forwarded_for character varying(9999), - country_iso_code character varying(10), - modified bigint DEFAULT public.now_as_millis() NOT NULL, - show_translation_activated boolean, - subscribe_email character varying(256) -); - - --- --- Name: permanentcookiezidjoins; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.permanentcookiezidjoins ( - zid integer NOT NULL, - cookie character varying(32), - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: polismath_mod_claims; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.polismath_mod_claims ( - n integer NOT NULL, - created bigint DEFAULT public.now_as_millis() NOT NULL -); - - --- --- Name: pwreset_tokens; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.pwreset_tokens ( - token character varying(100), - uid integer, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: queue; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.queue ( - itemid integer NOT NULL, - is_done boolean DEFAULT false NOT NULL -); - - --- --- Name: report_comment_selections; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.report_comment_selections ( - rid bigint NOT NULL, - tid integer NOT NULL, - selection smallint NOT NULL, - modified bigint DEFAULT public.now_as_millis(), - zid integer NOT NULL -); - - --- --- Name: reports; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.reports ( - rid bigint NOT NULL, - report_id character varying(300) NOT NULL, - zid integer NOT NULL, - created bigint DEFAULT public.now_as_millis(), - modified bigint DEFAULT public.now_as_millis(), - label_x_neg character varying(999), - label_y_neg character varying(999), - label_y_pos character varying(999), - label_x_pos character varying(999), - label_group_0 character varying(999), - label_group_1 character varying(999), - label_group_2 character varying(999), - label_group_3 character varying(999), - label_group_4 character varying(999), - label_group_5 character varying(999), - label_group_6 character varying(999), - label_group_7 character varying(999), - label_group_8 character varying(999), - label_group_9 character varying(999), - report_name character varying(999), - mod_level smallint DEFAULT '-2'::integer NOT NULL -); - - --- --- Name: reports_rid_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.reports_rid_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: reports_rid_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.reports_rid_seq OWNED BY public.reports.rid; - - --- --- Name: site_domain_whitelist; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.site_domain_whitelist ( - site_id character varying(256) NOT NULL, - domain_whitelist character varying(999), - modified bigint DEFAULT public.now_as_millis() NOT NULL, - created bigint DEFAULT public.now_as_millis() NOT NULL, - domain_whitelist_override_key character varying(999) -); - - --- --- Name: slack_participants_waiting_for_comments; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.slack_participants_waiting_for_comments ( - zid integer NOT NULL, - uid integer NOT NULL, - slack_team character varying(20) NOT NULL, - slack_user_id character varying(20) NOT NULL, - slack_channel_id character varying(20) NOT NULL, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: slack_state_heap; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.slack_state_heap ( - slack_team character varying(20) NOT NULL, - slack_user_id character varying(20) NOT NULL, - heap_data jsonb NOT NULL, - modified bigint DEFAULT public.now_as_millis(), - created bigint DEFAULT public.now_as_millis(), - active boolean DEFAULT true NOT NULL, - expires bigint DEFAULT 0 -); - - --- --- Name: slack_state_stack; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.slack_state_stack ( - slack_team character varying(20) NOT NULL, - slack_user_id character varying(20) NOT NULL, - stack_data jsonb NOT NULL, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: slack_team_tokens; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.slack_team_tokens ( - slack_team character varying(20) NOT NULL, - slack_bot_access_token character varying(200) NOT NULL, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: social_settings; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.social_settings ( - uid integer NOT NULL, - polis_pic character varying(3000) -); - - --- --- Name: stars; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.stars ( - zid integer NOT NULL, - pid integer NOT NULL, - tid integer NOT NULL, - starred integer NOT NULL, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: suzinvites; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.suzinvites ( - owner integer NOT NULL, - zid integer NOT NULL, - xid text NOT NULL, - created bigint DEFAULT public.now_as_millis(), - suzinvite character varying(32), - uid integer, - modified bigint DEFAULT public.now_as_millis() -); - - --- --- Name: topic_agenda_selections; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.topic_agenda_selections ( - zid integer NOT NULL, - pid integer NOT NULL, - archetypal_selections jsonb DEFAULT '[]'::jsonb NOT NULL, - delphi_job_id text, - total_selections integer DEFAULT 0 NOT NULL, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP -); - - --- --- Name: TABLE topic_agenda_selections; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON TABLE public.topic_agenda_selections IS 'Stores user topic agenda selections as archetypal comments that persist across Delphi runs'; - - --- --- Name: COLUMN topic_agenda_selections.zid; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.topic_agenda_selections.zid IS 'Conversation ID (foreign key to conversations)'; - - --- --- Name: COLUMN topic_agenda_selections.pid; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.topic_agenda_selections.pid IS 'Participant ID (foreign key to participants)'; - - --- --- Name: COLUMN topic_agenda_selections.archetypal_selections; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.topic_agenda_selections.archetypal_selections IS 'JSON array of selected topics with their archetypal comments'; - - --- --- Name: COLUMN topic_agenda_selections.delphi_job_id; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.topic_agenda_selections.delphi_job_id IS 'ID of the Delphi job that generated the topics'; - - --- --- Name: COLUMN topic_agenda_selections.total_selections; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.topic_agenda_selections.total_selections IS 'Total number of topics selected by the user'; - - --- --- Name: trashes; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.trashes ( - zid integer NOT NULL, - pid integer NOT NULL, - tid integer NOT NULL, - trashed integer NOT NULL, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: treevite_invites; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.treevite_invites ( - id bigint NOT NULL, - zid integer NOT NULL, - wave_id bigint NOT NULL, - parent_invite_id bigint, - invite_code character varying(64) NOT NULL, - status smallint DEFAULT 0 NOT NULL, - invite_owner_pid integer, - invite_used_by_pid integer, - invite_used_at timestamp with time zone, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - CONSTRAINT treevite_invites_status_check CHECK ((status = ANY (ARRAY[0, 1, 2, 3]))) -); - - --- --- Name: TABLE treevite_invites; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON TABLE public.treevite_invites IS 'Per-invite records for Treevite, including ownership, usage, and parent-child edges'; - - --- --- Name: COLUMN treevite_invites.invite_code; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.treevite_invites.invite_code IS 'Code shared by participants to grant access; unique per conversation'; - - --- --- Name: COLUMN treevite_invites.status; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.treevite_invites.status IS '0=unused, 1=used, 2=revoked, 3=expired'; - - --- --- Name: COLUMN treevite_invites.invite_owner_pid; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.treevite_invites.invite_owner_pid IS 'PID of participant who owns/distributes this invite (NULL for root invites)'; - - --- --- Name: COLUMN treevite_invites.invite_used_by_pid; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.treevite_invites.invite_used_by_pid IS 'PID of participant who consumed the invite (NULL until used)'; - - --- --- Name: treevite_invites_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.treevite_invites_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: treevite_invites_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.treevite_invites_id_seq OWNED BY public.treevite_invites.id; - - --- --- Name: treevite_login_codes; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.treevite_login_codes ( - id bigint NOT NULL, - zid integer NOT NULL, - pid integer NOT NULL, - login_code_hash text NOT NULL, - login_code_fingerprint character varying(128) NOT NULL, - login_code_lookup character varying(128), - fp_kid smallint DEFAULT 1 NOT NULL, - revoked boolean DEFAULT false NOT NULL, - expires_at timestamp with time zone, - last_used_at timestamp with time zone, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP -); - - --- --- Name: TABLE treevite_login_codes; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON TABLE public.treevite_login_codes IS 'Per-participant Treevite login codes: salted hash for verification plus HMAC fingerprint for lookup'; - - --- --- Name: COLUMN treevite_login_codes.login_code_hash; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.treevite_login_codes.login_code_hash IS 'Slow salted hash (argon2/bcrypt) of the participant login code; the raw code is never stored'; - - --- --- Name: COLUMN treevite_login_codes.login_code_fingerprint; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.treevite_login_codes.login_code_fingerprint IS 'Indexable HMAC-derived fingerprint scoped by conversation for fast lookup'; - - --- --- Name: COLUMN treevite_login_codes.login_code_lookup; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.treevite_login_codes.login_code_lookup IS 'Peppered SHA-256 of login_code for O(1) lookup; verify with bcrypt hash after lookup'; - - --- --- Name: treevite_login_codes_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.treevite_login_codes_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: treevite_login_codes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.treevite_login_codes_id_seq OWNED BY public.treevite_login_codes.id; - - --- --- Name: treevite_waves; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.treevite_waves ( - id bigint NOT NULL, - zid integer NOT NULL, - wave integer NOT NULL, - parent_wave integer, - size integer, - invites_per_user integer NOT NULL, - owner_invites integer DEFAULT 0 NOT NULL, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - CONSTRAINT treevite_waves_invites_per_user_check CHECK ((invites_per_user >= 0)), - CONSTRAINT treevite_waves_not_both_zero CHECK (((invites_per_user > 0) OR (owner_invites > 0))), - CONSTRAINT treevite_waves_owner_invites_check CHECK ((owner_invites >= 0)), - CONSTRAINT treevite_waves_parent_wave_check CHECK (((parent_wave IS NULL) OR (parent_wave >= 0))), - CONSTRAINT treevite_waves_size_check CHECK (((size IS NULL) OR (size >= 0))), - CONSTRAINT treevite_waves_wave_check CHECK ((wave >= 1)) -); - - --- --- Name: TABLE treevite_waves; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON TABLE public.treevite_waves IS 'Per-wave configuration and summary for Treevite invites'; - - --- --- Name: COLUMN treevite_waves.wave; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.treevite_waves.wave IS 'Wave number (1-based); unique per conversation'; - - --- --- Name: COLUMN treevite_waves.parent_wave; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.treevite_waves.parent_wave IS 'Parent wave number for deriving next wave; 0 for root, NULL means default to greatest existing wave for this zid or 0 if none'; - - --- --- Name: COLUMN treevite_waves.size; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.treevite_waves.size IS 'Optional cached size of the wave; derived as (parent_size or 1) * invites_per_user + owner_invites'; - - --- --- Name: COLUMN treevite_waves.invites_per_user; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.treevite_waves.invites_per_user IS 'Number of invites granted to each participant in this wave'; - - --- --- Name: COLUMN treevite_waves.owner_invites; Type: COMMENT; Schema: public; Owner: - --- - -COMMENT ON COLUMN public.treevite_waves.owner_invites IS 'Number of owner-controlled invites added to this wave'; - - --- --- Name: treevite_waves_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.treevite_waves_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: treevite_waves_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.treevite_waves_id_seq OWNED BY public.treevite_waves.id; - - --- --- Name: twitter_users; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.twitter_users ( - uid integer NOT NULL, - twitter_user_id bigint NOT NULL, - screen_name character varying(999) NOT NULL, - followers_count integer NOT NULL, - friends_count integer NOT NULL, - verified boolean NOT NULL, - profile_image_url_https character varying(9999), - modified bigint DEFAULT public.now_as_millis() NOT NULL, - created bigint DEFAULT public.now_as_millis() NOT NULL, - location character varying(9999), - response json, - name character varying(9999) -); - - --- --- Name: upvotes; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.upvotes ( - uid integer, - zid integer -); - - --- --- Name: users; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.users ( - uid integer NOT NULL, - hname character varying(746), - pwhash character varying(128), - username character varying(128), - email character varying(256), - is_owner boolean DEFAULT false, - zinvite character varying(300), - oinvite character varying(300), - created bigint DEFAULT public.now_as_millis(), - tut smallint DEFAULT 0, - site_id character varying(256) DEFAULT public.random_polis_site_id() NOT NULL, - site_owner boolean DEFAULT true, - test boolean DEFAULT false -); - - --- --- Name: users_uid_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.users_uid_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: users_uid_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.users_uid_seq OWNED BY public.users.uid; - - --- --- Name: votes_latest_unique; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.votes_latest_unique ( - zid integer NOT NULL, - pid integer NOT NULL, - tid integer NOT NULL, - vote smallint, - weight_x_32767 smallint DEFAULT 0, - modified bigint DEFAULT public.now_as_millis() -); - - --- --- Name: worker_tasks; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.worker_tasks ( - task_data jsonb NOT NULL, - created bigint DEFAULT public.now_as_millis(), - finished_time bigint, - task_type text, - math_env character varying(999) NOT NULL, - task_bucket bigint, - attempts smallint DEFAULT 0 NOT NULL -); - - --- --- Name: xid_whitelist; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.xid_whitelist ( - owner integer NOT NULL, - xid text NOT NULL, - created bigint DEFAULT public.now_as_millis() -); - - --- --- Name: xids; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.xids ( - uid integer NOT NULL, - owner integer NOT NULL, - xid text NOT NULL, - created bigint DEFAULT public.now_as_millis(), - x_profile_image_url character varying(3000), - x_name character varying(746), - modified bigint DEFAULT public.now_as_millis() NOT NULL, - x_email character varying(256) -); - - --- --- Name: zinvites; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.zinvites ( - zid integer NOT NULL, - zinvite character varying(300) NOT NULL, - created bigint DEFAULT public.now_as_millis(), - uuid uuid -); - - --- --- Name: contexts context_id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.contexts ALTER COLUMN context_id SET DEFAULT nextval('public.contexts_context_id_seq'::regclass); - - --- --- Name: conversations zid; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.conversations ALTER COLUMN zid SET DEFAULT nextval('public.conversations_zid_seq'::regclass); - - --- --- Name: courses course_id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.courses ALTER COLUMN course_id SET DEFAULT nextval('public.courses_course_id_seq'::regclass); - - --- --- Name: participant_metadata_answers pmaid; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participant_metadata_answers ALTER COLUMN pmaid SET DEFAULT nextval('public.participant_metadata_answers_pmaid_seq'::regclass); - - --- --- Name: participant_metadata_questions pmqid; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participant_metadata_questions ALTER COLUMN pmqid SET DEFAULT nextval('public.participant_metadata_questions_pmqid_seq'::regclass); - - --- --- Name: reports rid; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.reports ALTER COLUMN rid SET DEFAULT nextval('public.reports_rid_seq'::regclass); - - --- --- Name: treevite_invites id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_invites ALTER COLUMN id SET DEFAULT nextval('public.treevite_invites_id_seq'::regclass); - - --- --- Name: treevite_login_codes id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_login_codes ALTER COLUMN id SET DEFAULT nextval('public.treevite_login_codes_id_seq'::regclass); - - --- --- Name: treevite_waves id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_waves ALTER COLUMN id SET DEFAULT nextval('public.treevite_waves_id_seq'::regclass); - - --- --- Name: users uid; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.users ALTER COLUMN uid SET DEFAULT nextval('public.users_uid_seq'::regclass); - - --- --- Name: apikeysndvweifu apikeysndvweifu_apikey_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.apikeysndvweifu - ADD CONSTRAINT apikeysndvweifu_apikey_key UNIQUE (apikey); - - --- --- Name: auth_tokens auth_tokens_token_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.auth_tokens - ADD CONSTRAINT auth_tokens_token_key UNIQUE (token); - - --- --- Name: beta beta_email_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.beta - ADD CONSTRAINT beta_email_key UNIQUE (email); - - --- --- Name: comment_translations comment_translations_zid_tid_src_lang_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.comment_translations - ADD CONSTRAINT comment_translations_zid_tid_src_lang_key UNIQUE (zid, tid, src, lang); - - --- --- Name: comments comments_tid_unique_constraint; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.comments - ADD CONSTRAINT comments_tid_unique_constraint UNIQUE (zid, tid); - - --- --- Name: comments comments_txt_unique_constraint; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.comments - ADD CONSTRAINT comments_txt_unique_constraint UNIQUE (zid, txt); - - --- --- Name: conversation_invite_codes conversation_invite_codes_code_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.conversation_invite_codes - ADD CONSTRAINT conversation_invite_codes_code_key UNIQUE (code); - - --- --- Name: conversation_subscriptions conversation_subscriptions_zid_uid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.conversation_subscriptions - ADD CONSTRAINT conversation_subscriptions_zid_uid_key UNIQUE (zid, uid); - - --- --- Name: conversation_translations conversation_translations_zid_src_lang_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.conversation_translations - ADD CONSTRAINT conversation_translations_zid_src_lang_key UNIQUE (zid, src, lang); - - --- --- Name: conversations conversations_zid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.conversations - ADD CONSTRAINT conversations_zid_key UNIQUE (zid); - - --- --- Name: courses courses_course_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.courses - ADD CONSTRAINT courses_course_id_key UNIQUE (course_id); - - --- --- Name: demographic_data demographic_data_unique_uid; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.demographic_data - ADD CONSTRAINT demographic_data_unique_uid UNIQUE (uid); - - --- --- Name: einvites einvites_einvite_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.einvites - ADD CONSTRAINT einvites_einvite_key UNIQUE (einvite); - - --- --- Name: users email_unique_check; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.users - ADD CONSTRAINT email_unique_check UNIQUE (email); - - --- --- Name: email_validations email_validations_email_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.email_validations - ADD CONSTRAINT email_validations_email_key UNIQUE (email); - - --- --- Name: facebook_users facebook_users_fb_user_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.facebook_users - ADD CONSTRAINT facebook_users_fb_user_id_key UNIQUE (fb_user_id); - - --- --- Name: facebook_users facebook_users_uid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.facebook_users - ADD CONSTRAINT facebook_users_uid_key UNIQUE (uid); - - --- --- Name: jianiuevyew jianiuevyew_uid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.jianiuevyew - ADD CONSTRAINT jianiuevyew_uid_key UNIQUE (uid); - - --- --- Name: math_bidtopid math_bidtopid_zid_math_env_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.math_bidtopid - ADD CONSTRAINT math_bidtopid_zid_math_env_key UNIQUE (zid, math_env); - - --- --- Name: math_cache math_cache_zid_math_env_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.math_cache - ADD CONSTRAINT math_cache_zid_math_env_key UNIQUE (zid, math_env); - - --- --- Name: math_exportstatus math_exportstatus_zid_math_env_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.math_exportstatus - ADD CONSTRAINT math_exportstatus_zid_math_env_key UNIQUE (zid, math_env); - - --- --- Name: math_main math_main_zid_math_env_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.math_main - ADD CONSTRAINT math_main_zid_math_env_key UNIQUE (zid, math_env); - - --- --- Name: math_profile math_profile_zid_math_env_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.math_profile - ADD CONSTRAINT math_profile_zid_math_env_key UNIQUE (zid, math_env); - - --- --- Name: math_ptptstats math_ptptstats_zid_math_env_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.math_ptptstats - ADD CONSTRAINT math_ptptstats_zid_math_env_key UNIQUE (zid, math_env); - - --- --- Name: math_report_correlationmatrix math_report_correlationmatrix_rid_math_env_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.math_report_correlationmatrix - ADD CONSTRAINT math_report_correlationmatrix_rid_math_env_key UNIQUE (rid, math_env); - - --- --- Name: math_results_dev01 math_results_dev01_zid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.math_results_dev01 - ADD CONSTRAINT math_results_dev01_zid_key UNIQUE (zid); - - --- --- Name: math_ticks math_ticks_zid_math_env_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.math_ticks - ADD CONSTRAINT math_ticks_zid_math_env_key UNIQUE (zid, math_env); - - --- --- Name: minvites minvites_minvite_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.minvites - ADD CONSTRAINT minvites_minvite_key UNIQUE (minvite); - - --- --- Name: moderators moderators_zid_uid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.moderators - ADD CONSTRAINT moderators_zid_uid_key UNIQUE (zid, uid); - - --- --- Name: notification_tasks notification_tasks_zid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.notification_tasks - ADD CONSTRAINT notification_tasks_zid_key UNIQUE (zid); - - --- --- Name: nyt_users nyt_users_nyt_user_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.nyt_users - ADD CONSTRAINT nyt_users_nyt_user_id_key UNIQUE (nyt_user_id); - - --- --- Name: nyt_users nyt_users_uid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.nyt_users - ADD CONSTRAINT nyt_users_uid_key UNIQUE (uid); - - --- --- Name: oidc_user_mappings oidc_user_mappings_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.oidc_user_mappings - ADD CONSTRAINT oidc_user_mappings_pkey PRIMARY KEY (oidc_sub); - - --- --- Name: oidc_user_mappings oidc_user_mappings_uid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.oidc_user_mappings - ADD CONSTRAINT oidc_user_mappings_uid_key UNIQUE (uid); - - --- --- Name: oinvites oinvites_oinvite_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.oinvites - ADD CONSTRAINT oinvites_oinvite_key UNIQUE (oinvite); - - --- --- Name: page_ids page_ids_site_id_page_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.page_ids - ADD CONSTRAINT page_ids_site_id_page_id_key UNIQUE (site_id, page_id); - - --- --- Name: participant_locations participant_locations_zid_uid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participant_locations - ADD CONSTRAINT participant_locations_zid_uid_key UNIQUE (zid, uid); - - --- --- Name: participant_metadata_answers participant_metadata_answers_pmaid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participant_metadata_answers - ADD CONSTRAINT participant_metadata_answers_pmaid_key UNIQUE (pmaid); - - --- --- Name: participant_metadata_answers participant_metadata_answers_pmqid_zid_value_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participant_metadata_answers - ADD CONSTRAINT participant_metadata_answers_pmqid_zid_value_key UNIQUE (pmqid, zid, value); - - --- --- Name: participant_metadata_choices participant_metadata_choices_zid_pid_pmqid_pmaid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participant_metadata_choices - ADD CONSTRAINT participant_metadata_choices_zid_pid_pmqid_pmaid_key UNIQUE (zid, pid, pmqid, pmaid); - - --- --- Name: participant_metadata_questions participant_metadata_questions_pmqid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participant_metadata_questions - ADD CONSTRAINT participant_metadata_questions_pmqid_key UNIQUE (pmqid); - - --- --- Name: participant_metadata_questions participant_metadata_questions_zid_key_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participant_metadata_questions - ADD CONSTRAINT participant_metadata_questions_zid_key_key UNIQUE (zid, key); - - --- --- Name: participants_extended participants_extended_zid_uid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participants_extended - ADD CONSTRAINT participants_extended_zid_uid_key UNIQUE (zid, uid); - - --- --- Name: participants participants_zid_pid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participants - ADD CONSTRAINT participants_zid_pid_key UNIQUE (zid, pid); - - --- --- Name: participants participants_zid_uid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participants - ADD CONSTRAINT participants_zid_uid_key UNIQUE (zid, uid); - - --- --- Name: permanentcookiezidjoins permanentcookiezidjoins_zid_cookie_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.permanentcookiezidjoins - ADD CONSTRAINT permanentcookiezidjoins_zid_cookie_key UNIQUE (zid, cookie); - - --- --- Name: pwreset_tokens pwreset_tokens_token_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.pwreset_tokens - ADD CONSTRAINT pwreset_tokens_token_key UNIQUE (token); - - --- --- Name: queue queue_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.queue - ADD CONSTRAINT queue_pkey PRIMARY KEY (itemid); - - --- --- Name: report_comment_selections report_comment_selections_rid_tid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.report_comment_selections - ADD CONSTRAINT report_comment_selections_rid_tid_key UNIQUE (rid, tid); - - --- --- Name: reports reports_report_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.reports - ADD CONSTRAINT reports_report_id_key UNIQUE (report_id); - - --- --- Name: reports reports_rid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.reports - ADD CONSTRAINT reports_rid_key UNIQUE (rid); - - --- --- Name: slack_participants_waiting_for_comments slack_participants_waiting_for_slack_team_slack_user_id_zid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.slack_participants_waiting_for_comments - ADD CONSTRAINT slack_participants_waiting_for_slack_team_slack_user_id_zid_key UNIQUE (slack_team, slack_user_id, zid); - - --- --- Name: slack_state_heap slack_state_heap_slack_team_slack_user_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.slack_state_heap - ADD CONSTRAINT slack_state_heap_slack_team_slack_user_id_key UNIQUE (slack_team, slack_user_id); - - --- --- Name: slack_state_stack slack_state_stack_slack_team_slack_user_id_created_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.slack_state_stack - ADD CONSTRAINT slack_state_stack_slack_team_slack_user_id_created_key UNIQUE (slack_team, slack_user_id, created); - - --- --- Name: slack_team_tokens slack_team_tokens_slack_team_slack_bot_access_token_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.slack_team_tokens - ADD CONSTRAINT slack_team_tokens_slack_team_slack_bot_access_token_key UNIQUE (slack_team, slack_bot_access_token); - - --- --- Name: social_settings social_settings_uid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.social_settings - ADD CONSTRAINT social_settings_uid_key UNIQUE (uid); - - --- --- Name: suzinvites suzinvites_suzinvite_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.suzinvites - ADD CONSTRAINT suzinvites_suzinvite_key UNIQUE (suzinvite); - - --- --- Name: topic_agenda_selections topic_agenda_selections_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.topic_agenda_selections - ADD CONSTRAINT topic_agenda_selections_pkey PRIMARY KEY (zid, pid); - - --- --- Name: treevite_invites treevite_invites_code_unique; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_invites - ADD CONSTRAINT treevite_invites_code_unique UNIQUE (zid, invite_code); - - --- --- Name: treevite_invites treevite_invites_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_invites - ADD CONSTRAINT treevite_invites_pkey PRIMARY KEY (id); - - --- --- Name: treevite_login_codes treevite_login_codes_fp_unique; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_login_codes - ADD CONSTRAINT treevite_login_codes_fp_unique UNIQUE (zid, login_code_fingerprint); - - --- --- Name: treevite_login_codes treevite_login_codes_lookup_unique; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_login_codes - ADD CONSTRAINT treevite_login_codes_lookup_unique UNIQUE (zid, login_code_lookup); - - --- --- Name: treevite_login_codes treevite_login_codes_pid_unique; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_login_codes - ADD CONSTRAINT treevite_login_codes_pid_unique UNIQUE (zid, pid); - - --- --- Name: treevite_login_codes treevite_login_codes_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_login_codes - ADD CONSTRAINT treevite_login_codes_pkey PRIMARY KEY (id); - - --- --- Name: treevite_waves treevite_waves_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_waves - ADD CONSTRAINT treevite_waves_pkey PRIMARY KEY (id); - - --- --- Name: treevite_waves treevite_waves_zid_wave_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_waves - ADD CONSTRAINT treevite_waves_zid_wave_key UNIQUE (zid, wave); - - --- --- Name: twitter_users twitter_users_twitter_user_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.twitter_users - ADD CONSTRAINT twitter_users_twitter_user_id_key UNIQUE (twitter_user_id); - - --- --- Name: twitter_users twitter_users_uid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.twitter_users - ADD CONSTRAINT twitter_users_uid_key UNIQUE (uid); - - --- --- Name: upvotes upvotes_uid_zid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.upvotes - ADD CONSTRAINT upvotes_uid_zid_key UNIQUE (uid, zid); - - --- --- Name: users users_uid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.users - ADD CONSTRAINT users_uid_key UNIQUE (uid); - - --- --- Name: votes_latest_unique votes_latest_unique_zid_pid_tid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.votes_latest_unique - ADD CONSTRAINT votes_latest_unique_zid_pid_tid_key UNIQUE (zid, pid, tid); - - --- --- Name: xid_whitelist xid_whitelist_owner_xid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.xid_whitelist - ADD CONSTRAINT xid_whitelist_owner_xid_key UNIQUE (owner, xid); - - --- --- Name: xids xids_owner_xid_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.xids - ADD CONSTRAINT xids_owner_xid_key UNIQUE (owner, xid); - - --- --- Name: xids xids_unique_owner_uid_constraint; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.xids - ADD CONSTRAINT xids_unique_owner_uid_constraint UNIQUE (owner, uid); - - --- --- Name: zinvites zinvites_zinvite_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.zinvites - ADD CONSTRAINT zinvites_zinvite_key UNIQUE (zinvite); - - --- --- Name: apikeysndvweifu_apikey_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX apikeysndvweifu_apikey_idx ON public.apikeysndvweifu USING btree (apikey); - - --- --- Name: apikeysndvweifu_uid_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX apikeysndvweifu_uid_idx ON public.apikeysndvweifu USING btree (uid); - - --- --- Name: auth_tokens_token_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX auth_tokens_token_idx ON public.auth_tokens USING btree (token); - - --- --- Name: comment_translations_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX comment_translations_idx ON public.comment_translations USING btree (zid, tid); - - --- --- Name: comments_zid_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX comments_zid_idx ON public.comments USING btree (zid); - - --- --- Name: conversation_translations_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX conversation_translations_idx ON public.conversation_translations USING btree (zid); - - --- --- Name: conversations_owner_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX conversations_owner_idx ON public.conversations USING btree (owner); - - --- --- Name: conversations_zid_index; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX conversations_zid_index ON public.conversations USING btree (zid); - - --- --- Name: course_id_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX course_id_idx ON public.courses USING btree (course_id); - - --- --- Name: idx_oidc_mappings_uid; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_oidc_mappings_uid ON public.oidc_user_mappings USING btree (uid); - - --- --- Name: idx_topic_agenda_selections_created_at; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_topic_agenda_selections_created_at ON public.topic_agenda_selections USING btree (created_at); - - --- --- Name: idx_topic_agenda_selections_delphi_job_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_topic_agenda_selections_delphi_job_id ON public.topic_agenda_selections USING btree (delphi_job_id); - - --- --- Name: idx_topic_agenda_selections_pid; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_topic_agenda_selections_pid ON public.topic_agenda_selections USING btree (pid); - - --- --- Name: idx_topic_agenda_selections_zid; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_topic_agenda_selections_zid ON public.topic_agenda_selections USING btree (zid); - - --- --- Name: idx_treevite_invites_code; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_treevite_invites_code ON public.treevite_invites USING btree (invite_code); - - --- --- Name: idx_treevite_invites_owner_pid; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_treevite_invites_owner_pid ON public.treevite_invites USING btree (invite_owner_pid); - - --- --- Name: idx_treevite_invites_parent; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_treevite_invites_parent ON public.treevite_invites USING btree (parent_invite_id); - - --- --- Name: idx_treevite_invites_used_by_pid; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_treevite_invites_used_by_pid ON public.treevite_invites USING btree (invite_used_by_pid); - - --- --- Name: idx_treevite_invites_wave_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_treevite_invites_wave_id ON public.treevite_invites USING btree (wave_id); - - --- --- Name: idx_treevite_invites_zid; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_treevite_invites_zid ON public.treevite_invites USING btree (zid); - - --- --- Name: idx_treevite_invites_zid_status; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_treevite_invites_zid_status ON public.treevite_invites USING btree (zid, status); - - --- --- Name: idx_treevite_login_codes_fp; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_treevite_login_codes_fp ON public.treevite_login_codes USING btree (login_code_fingerprint); - - --- --- Name: idx_treevite_login_codes_lookup; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_treevite_login_codes_lookup ON public.treevite_login_codes USING btree (zid, login_code_lookup); - - --- --- Name: idx_treevite_login_codes_pid; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_treevite_login_codes_pid ON public.treevite_login_codes USING btree (pid); - - --- --- Name: idx_treevite_login_codes_zid; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_treevite_login_codes_zid ON public.treevite_login_codes USING btree (zid); - - --- --- Name: idx_treevite_waves_parent; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_treevite_waves_parent ON public.treevite_waves USING btree (zid, parent_wave); - - --- --- Name: idx_treevite_waves_wave; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_treevite_waves_wave ON public.treevite_waves USING btree (wave); - - --- --- Name: idx_treevite_waves_zid; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_treevite_waves_zid ON public.treevite_waves USING btree (zid); - - --- --- Name: main_main_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX main_main_idx ON public.math_main USING btree (zid); - - --- --- Name: main_profile_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX main_profile_idx ON public.math_profile USING btree (zid); - - --- --- Name: math_bidtopid_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX math_bidtopid_idx ON public.math_bidtopid USING btree (zid); - - --- --- Name: math_cache_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX math_cache_idx ON public.math_cache USING btree (zid); - - --- --- Name: math_exportstatus_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX math_exportstatus_idx ON public.math_exportstatus USING btree (zid); - - --- --- Name: math_math_report_correlationmatrix_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX math_math_report_correlationmatrix_idx ON public.math_report_correlationmatrix USING btree (rid); - - --- --- Name: math_ptptstats_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX math_ptptstats_idx ON public.math_ptptstats USING btree (zid); - - --- --- Name: participants_conv_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX participants_conv_idx ON public.participants USING btree (zid); - - --- --- Name: participants_conv_uid_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX participants_conv_uid_idx ON public.participants USING btree (uid); - - --- --- Name: participants_uid_index; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX participants_uid_index ON public.participants USING btree (uid); - - --- --- Name: pwreset_tokens_token_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX pwreset_tokens_token_idx ON public.pwreset_tokens USING btree (token); - - --- --- Name: site_domain_whitelist_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX site_domain_whitelist_idx ON public.users USING btree (site_id); - - --- --- Name: suzinvites_owner_zid_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX suzinvites_owner_zid_idx ON public.suzinvites USING btree (owner, zid); - - --- --- Name: votes_latest_unique_zid_tid_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX votes_latest_unique_zid_tid_idx ON public.votes USING btree (zid, tid); - - --- --- Name: votes_zid_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX votes_zid_idx ON public.votes USING btree (zid); - - --- --- Name: votes_zid_pid_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX votes_zid_pid_idx ON public.votes USING btree (zid, pid); - - --- --- Name: xid_whitelist_owner_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX xid_whitelist_owner_idx ON public.xid_whitelist USING btree (owner); - - --- --- Name: xids_owner_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX xids_owner_idx ON public.xids USING btree (owner); - - --- --- Name: zinvites_zid_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX zinvites_zid_idx ON public.zinvites USING btree (zid); - - --- --- Name: votes on_vote_insert_update_unique_table; Type: RULE; Schema: public; Owner: - --- - -CREATE RULE on_vote_insert_update_unique_table AS - ON INSERT TO public.votes DO INSERT INTO public.votes_latest_unique (zid, pid, tid, vote, weight_x_32767, modified) - VALUES (new.zid, new.pid, new.tid, new.vote, new.weight_x_32767, new.created) ON CONFLICT(zid, pid, tid) DO UPDATE SET vote = excluded.vote, modified = excluded.modified; - - --- --- Name: participants pid_auto; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER pid_auto BEFORE INSERT ON public.participants FOR EACH ROW EXECUTE FUNCTION public.pid_auto(); - - --- --- Name: participants pid_auto_unlock; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER pid_auto_unlock AFTER INSERT ON public.participants FOR EACH ROW EXECUTE FUNCTION public.pid_auto_unlock(); - - --- --- Name: comments tid_auto; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER tid_auto BEFORE INSERT ON public.comments FOR EACH ROW EXECUTE FUNCTION public.tid_auto(); - - --- --- Name: comments tid_auto_unlock; Type: TRIGGER; Schema: public; Owner: - --- - -CREATE TRIGGER tid_auto_unlock AFTER INSERT ON public.comments FOR EACH ROW EXECUTE FUNCTION public.tid_auto_unlock(); - - --- --- Name: apikeysndvweifu apikeysndvweifu_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.apikeysndvweifu - ADD CONSTRAINT apikeysndvweifu_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: auth_tokens auth_tokens_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.auth_tokens - ADD CONSTRAINT auth_tokens_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: comment_translations comment_translations_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.comment_translations - ADD CONSTRAINT comment_translations_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: comments comments_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.comments - ADD CONSTRAINT comments_zid_fkey FOREIGN KEY (zid, pid) REFERENCES public.participants(zid, pid); - - --- --- Name: contexts contexts_creator_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.contexts - ADD CONSTRAINT contexts_creator_fkey FOREIGN KEY (creator) REFERENCES public.users(uid); - - --- --- Name: contributor_agreement_signatures contributer_agreement_signatures_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.contributor_agreement_signatures - ADD CONSTRAINT contributer_agreement_signatures_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: conversation_invite_codes conversation_invite_codes_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.conversation_invite_codes - ADD CONSTRAINT conversation_invite_codes_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: conversation_subscriptions conversation_subscriptions_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.conversation_subscriptions - ADD CONSTRAINT conversation_subscriptions_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: conversation_subscriptions conversation_subscriptions_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.conversation_subscriptions - ADD CONSTRAINT conversation_subscriptions_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: conversation_translations conversation_translations_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.conversation_translations - ADD CONSTRAINT conversation_translations_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: conversations conversations_actual_ownr_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.conversations - ADD CONSTRAINT conversations_actual_ownr_fkey FOREIGN KEY (org_id) REFERENCES public.users(uid); - - --- --- Name: conversations conversations_course_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.conversations - ADD CONSTRAINT conversations_course_id_fkey FOREIGN KEY (course_id) REFERENCES public.courses(course_id); - - --- --- Name: conversations conversations_owner_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.conversations - ADD CONSTRAINT conversations_owner_fkey FOREIGN KEY (owner) REFERENCES public.users(uid); - - --- --- Name: courses courses_owner_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.courses - ADD CONSTRAINT courses_owner_fkey FOREIGN KEY (owner) REFERENCES public.users(uid); - - --- --- Name: demographic_data demographic_data_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.demographic_data - ADD CONSTRAINT demographic_data_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: error_reports error_reports_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.error_reports - ADD CONSTRAINT error_reports_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: error_reports error_reports_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.error_reports - ADD CONSTRAINT error_reports_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: facebook_friends facebook_friends_friend_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.facebook_friends - ADD CONSTRAINT facebook_friends_friend_fkey FOREIGN KEY (friend) REFERENCES public.users(uid); - - --- --- Name: facebook_friends facebook_friends_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.facebook_friends - ADD CONSTRAINT facebook_friends_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: facebook_users facebook_users_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.facebook_users - ADD CONSTRAINT facebook_users_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: topic_agenda_selections fk_conversation; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.topic_agenda_selections - ADD CONSTRAINT fk_conversation FOREIGN KEY (zid) REFERENCES public.conversations(zid) ON DELETE CASCADE; - - --- --- Name: topic_agenda_selections fk_participant; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.topic_agenda_selections - ADD CONSTRAINT fk_participant FOREIGN KEY (zid, pid) REFERENCES public.participants(zid, pid) ON DELETE CASCADE; - - --- --- Name: inviters inviters_inviter_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.inviters - ADD CONSTRAINT inviters_inviter_uid_fkey FOREIGN KEY (inviter_uid) REFERENCES public.users(uid); - - --- --- Name: jianiuevyew jianiuevyew_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.jianiuevyew - ADD CONSTRAINT jianiuevyew_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: math_bidtopid math_bidtopid_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.math_bidtopid - ADD CONSTRAINT math_bidtopid_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: math_cache math_cache_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.math_cache - ADD CONSTRAINT math_cache_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: math_exportstatus math_exportstatus_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.math_exportstatus - ADD CONSTRAINT math_exportstatus_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: math_main math_main_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.math_main - ADD CONSTRAINT math_main_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: math_profile math_profile_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.math_profile - ADD CONSTRAINT math_profile_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: math_ptptstats math_ptptstats_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.math_ptptstats - ADD CONSTRAINT math_ptptstats_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: math_report_correlationmatrix math_report_correlationmatrix_rid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.math_report_correlationmatrix - ADD CONSTRAINT math_report_correlationmatrix_rid_fkey FOREIGN KEY (rid) REFERENCES public.reports(rid); - - --- --- Name: math_ticks math_ticks_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.math_ticks - ADD CONSTRAINT math_ticks_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: metrics metrics_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.metrics - ADD CONSTRAINT metrics_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: minvites minvites_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.minvites - ADD CONSTRAINT minvites_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: moderators moderators_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.moderators - ADD CONSTRAINT moderators_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: moderators moderators_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.moderators - ADD CONSTRAINT moderators_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: notification_tasks notification_tasks_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.notification_tasks - ADD CONSTRAINT notification_tasks_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: nyt_users nyt_users_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.nyt_users - ADD CONSTRAINT nyt_users_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: oidc_user_mappings oidc_user_mappings_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.oidc_user_mappings - ADD CONSTRAINT oidc_user_mappings_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid) ON DELETE CASCADE; - - --- --- Name: page_ids page_ids_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.page_ids - ADD CONSTRAINT page_ids_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: participant_locations participant_locations_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participant_locations - ADD CONSTRAINT participant_locations_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: participant_locations participant_locations_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participant_locations - ADD CONSTRAINT participant_locations_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: participant_metadata_answers participant_metadata_answers_pmqid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participant_metadata_answers - ADD CONSTRAINT participant_metadata_answers_pmqid_fkey FOREIGN KEY (pmqid) REFERENCES public.participant_metadata_questions(pmqid); - - --- --- Name: participant_metadata_answers participant_metadata_answers_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participant_metadata_answers - ADD CONSTRAINT participant_metadata_answers_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: participant_metadata_choices participant_metadata_choices_pmaid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participant_metadata_choices - ADD CONSTRAINT participant_metadata_choices_pmaid_fkey FOREIGN KEY (pmaid) REFERENCES public.participant_metadata_answers(pmaid); - - --- --- Name: participant_metadata_choices participant_metadata_choices_pmqid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participant_metadata_choices - ADD CONSTRAINT participant_metadata_choices_pmqid_fkey FOREIGN KEY (pmqid) REFERENCES public.participant_metadata_questions(pmqid); - - --- --- Name: participant_metadata_choices participant_metadata_choices_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participant_metadata_choices - ADD CONSTRAINT participant_metadata_choices_zid_fkey FOREIGN KEY (zid, pid) REFERENCES public.participants(zid, pid); - - --- --- Name: participant_metadata_questions participant_metadata_questions_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participant_metadata_questions - ADD CONSTRAINT participant_metadata_questions_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: participants_extended participants_extended_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participants_extended - ADD CONSTRAINT participants_extended_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: participants_extended participants_extended_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participants_extended - ADD CONSTRAINT participants_extended_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: participants participants_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participants - ADD CONSTRAINT participants_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: participants participants_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.participants - ADD CONSTRAINT participants_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: pwreset_tokens pwreset_tokens_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.pwreset_tokens - ADD CONSTRAINT pwreset_tokens_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: report_comment_selections report_comment_selections_rid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.report_comment_selections - ADD CONSTRAINT report_comment_selections_rid_fkey FOREIGN KEY (rid) REFERENCES public.reports(rid); - - --- --- Name: report_comment_selections report_comment_selections_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.report_comment_selections - ADD CONSTRAINT report_comment_selections_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: reports reports_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.reports - ADD CONSTRAINT reports_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: slack_participants_waiting_for_comments slack_participants_waiting_for_comments_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.slack_participants_waiting_for_comments - ADD CONSTRAINT slack_participants_waiting_for_comments_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: slack_participants_waiting_for_comments slack_participants_waiting_for_comments_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.slack_participants_waiting_for_comments - ADD CONSTRAINT slack_participants_waiting_for_comments_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: social_settings social_settings_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.social_settings - ADD CONSTRAINT social_settings_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: suzinvites suzinvites_owner_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.suzinvites - ADD CONSTRAINT suzinvites_owner_fkey FOREIGN KEY (owner) REFERENCES public.users(uid); - - --- --- Name: suzinvites suzinvites_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.suzinvites - ADD CONSTRAINT suzinvites_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: suzinvites suzinvites_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.suzinvites - ADD CONSTRAINT suzinvites_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: treevite_invites treevite_invites_owner_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_invites - ADD CONSTRAINT treevite_invites_owner_fkey FOREIGN KEY (zid, invite_owner_pid) REFERENCES public.participants(zid, pid); - - --- --- Name: treevite_invites treevite_invites_parent_invite_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_invites - ADD CONSTRAINT treevite_invites_parent_invite_id_fkey FOREIGN KEY (parent_invite_id) REFERENCES public.treevite_invites(id) ON DELETE SET NULL; - - --- --- Name: treevite_invites treevite_invites_used_by_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_invites - ADD CONSTRAINT treevite_invites_used_by_fkey FOREIGN KEY (zid, invite_used_by_pid) REFERENCES public.participants(zid, pid); - - --- --- Name: treevite_invites treevite_invites_wave_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_invites - ADD CONSTRAINT treevite_invites_wave_id_fkey FOREIGN KEY (wave_id) REFERENCES public.treevite_waves(id) ON DELETE CASCADE; - - --- --- Name: treevite_invites treevite_invites_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_invites - ADD CONSTRAINT treevite_invites_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid) ON DELETE CASCADE; - - --- --- Name: treevite_login_codes treevite_login_codes_participant_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_login_codes - ADD CONSTRAINT treevite_login_codes_participant_fkey FOREIGN KEY (zid, pid) REFERENCES public.participants(zid, pid) ON DELETE CASCADE; - - --- --- Name: treevite_waves treevite_waves_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.treevite_waves - ADD CONSTRAINT treevite_waves_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid) ON DELETE CASCADE; - - --- --- Name: twitter_users twitter_users_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.twitter_users - ADD CONSTRAINT twitter_users_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: upvotes upvotes_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.upvotes - ADD CONSTRAINT upvotes_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: upvotes upvotes_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.upvotes - ADD CONSTRAINT upvotes_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- Name: xid_whitelist xid_whitelist_owner_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.xid_whitelist - ADD CONSTRAINT xid_whitelist_owner_fkey FOREIGN KEY (owner) REFERENCES public.users(uid); - - --- --- Name: xids xids_owner_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.xids - ADD CONSTRAINT xids_owner_fkey FOREIGN KEY (owner) REFERENCES public.users(uid); - - --- --- Name: xids xids_uid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.xids - ADD CONSTRAINT xids_uid_fkey FOREIGN KEY (uid) REFERENCES public.users(uid); - - --- --- Name: zinvites zinvites_zid_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.zinvites - ADD CONSTRAINT zinvites_zid_fkey FOREIGN KEY (zid) REFERENCES public.conversations(zid); - - --- --- PostgreSQL database dump complete --- - -\unrestrict yi3q8cVeyPYXP7xpzDDwpdv2ts6MbtdVFjxHM5kY0sd3iTqn6dEoZ2y0ASHXmgp -