diff --git a/client/package-lock.json b/client/package-lock.json index 493be3b5..0f1ad5eb 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -33,7 +33,8 @@ "react-table": "^6.8.6", "redux": "^4.0.5", "redux-saga": "^1.1.3", - "semantic-ui-react": "^2.0.1" + "semantic-ui-react": "^2.0.1", + "three": "^0.138.3" }, "devDependencies": { "@babel/cli": "^7.12.8", @@ -54,6 +55,7 @@ "@types/react": "^16.14.2", "@types/react-dom": "^16.9.10", "@types/react-redux": "^7.1.11", + "@types/three": "^0.138.0", "@typescript-eslint/eslint-plugin": "^5.5.0", "@typescript-eslint/parser": "^5.5.0", "babel-jest": "^27.0.6", @@ -3716,6 +3718,12 @@ "@types/jest": "*" } }, + "node_modules/@types/three": { + "version": "0.138.0", + "resolved": "https://registry.npmjs.org/@types/three/-/three-0.138.0.tgz", + "integrity": "sha512-D8AoV7h2kbCfrv/DcebHOFh1WDwyus3HdooBkAwcBikXArdqnsQ38PQ85JCunnvun160oA9jz53GszF3zch3tg==", + "dev": true + }, "node_modules/@types/ws": { "version": "8.5.2", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.2.tgz", @@ -7710,10 +7718,9 @@ "dev": true }, "node_modules/grpc-bus": { - "version": "v1.0.1-dev5", + "version": "1.0.1-dev5", "resolved": "https://github.com/gabrielgrant/grpc-bus/releases/download/v1.0.1-dev5/grpc-bus-1.0.1-dev5.tgz", "integrity": "sha512-u5ZNfzrgFIPSltdbAzXE409wYGTzXEG2doyMCzmK86h8Ju2Ny7y1zxaXjtNmc9RBuIfaQsAHhOP0e4H+KL6RUg==", - "license": "MIT", "dependencies": { "lodash": "^4.0.0", "rxjs": "5.4.2" @@ -13860,6 +13867,11 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, + "node_modules/three": { + "version": "0.138.3", + "resolved": "https://registry.npmjs.org/three/-/three-0.138.3.tgz", + "integrity": "sha512-4t1cKC8gimNyJChJbaklg8W/qj3PpsLJUIFm5LIuAy/hVxxNm1ru2FGTSfbTSsuHmC/7ipsyuGKqrSAKLNtkzg==" + }, "node_modules/throat": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/throat/-/throat-6.0.1.tgz", @@ -18116,6 +18128,12 @@ "@types/jest": "*" } }, + "@types/three": { + "version": "0.138.0", + "resolved": "https://registry.npmjs.org/@types/three/-/three-0.138.0.tgz", + "integrity": "sha512-D8AoV7h2kbCfrv/DcebHOFh1WDwyus3HdooBkAwcBikXArdqnsQ38PQ85JCunnvun160oA9jz53GszF3zch3tg==", + "dev": true + }, "@types/ws": { "version": "8.5.2", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.2.tgz", @@ -25815,6 +25833,11 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, + "three": { + "version": "0.138.3", + "resolved": "https://registry.npmjs.org/three/-/three-0.138.3.tgz", + "integrity": "sha512-4t1cKC8gimNyJChJbaklg8W/qj3PpsLJUIFm5LIuAy/hVxxNm1ru2FGTSfbTSsuHmC/7ipsyuGKqrSAKLNtkzg==" + }, "throat": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/throat/-/throat-6.0.1.tgz", diff --git a/client/package.json b/client/package.json index 6029878c..74d4e49f 100644 --- a/client/package.json +++ b/client/package.json @@ -52,7 +52,8 @@ "react-table": "^6.8.6", "redux": "^4.0.5", "redux-saga": "^1.1.3", - "semantic-ui-react": "^2.0.1" + "semantic-ui-react": "^2.0.1", + "three": "^0.138.3" }, "devDependencies": { "@babel/cli": "^7.12.8", @@ -73,6 +74,7 @@ "@types/react": "^16.14.2", "@types/react-dom": "^16.9.10", "@types/react-redux": "^7.1.11", + "@types/three": "^0.138.0", "@typescript-eslint/eslint-plugin": "^5.5.0", "@typescript-eslint/parser": "^5.5.0", "babel-jest": "^27.0.6", @@ -152,7 +154,10 @@ "ecmaVersion": 2018, "sourceType": "module" }, - "plugins": ["react", "@typescript-eslint"], + "plugins": [ + "react", + "@typescript-eslint" + ], "rules": { "prettier/prettier": "error", "no-unused-vars": [ @@ -170,9 +175,17 @@ "no-var": "warn", "react/prop-types": "off", "react/jsx-key": "off", - "eqeqeq": ["error", "always"], - "curly": ["error", "all"], - "no-bitwise": ["error"] + "eqeqeq": [ + "error", + "always" + ], + "curly": [ + "error", + "all" + ], + "no-bitwise": [ + "error" + ] }, "settings": { "react": { diff --git a/client/src/api/auth.ts b/client/src/api/auth.ts index 85721a35..25b2c297 100644 --- a/client/src/api/auth.ts +++ b/client/src/api/auth.ts @@ -177,7 +177,8 @@ const decodeProvider = (data: unknown): Result => { 'url' in data && typeof (data as { id: unknown }).id === 'number' && typeof (data as { name: unknown }).name === 'string' && - typeof (data as { icon: unknown }).icon === 'string' && + (typeof (data as { icon: unknown }).icon === 'string' || + (data as { icon: unknown }).icon === null) && typeof (data as { url: unknown }).url === 'string' ) { const provider: Provider = { diff --git a/client/src/api/project.ts b/client/src/api/project.ts index 8a2ec8e5..27d2a10f 100644 --- a/client/src/api/project.ts +++ b/client/src/api/project.ts @@ -185,3 +185,27 @@ export async function makeProject( return error(`Cannot create project: ${JSON.stringify(err)}`); } } + +export type Coordinate = { + x: number; + y: number; +}; + +export async function getCoordinates( + project: string, + dataset: number +): Promise> { + const url = new URL(API_URL + 'data/dataset'); + url.search = new URLSearchParams({ + project, + dataset: `${dataset}`, + }).toString(); + const response = await fetch(url.toString(), { + method: 'GET', + headers: { + Accept: 'application/json', + }, + }); + + return (await response.json()) as Array; +} diff --git a/client/src/components/Viewer/Viewer.tsx b/client/src/components/Viewer/Viewer.tsx new file mode 100644 index 00000000..90b9d4d6 --- /dev/null +++ b/client/src/components/Viewer/Viewer.tsx @@ -0,0 +1,254 @@ +import React, { useEffect, useRef } from 'react'; +import * as THREE from 'three'; +import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls'; +import { useDispatch, useSelector } from 'react-redux'; +import { Coordinate } from '../../api'; +import { RootState } from '../../redux/reducers'; +import { getCoordinates } from '../../redux/actions'; +import * as R from 'ramda'; + +export type ViewerProps = { + project: string; + dataset: number; +}; + +const initGraphics = () => { + const scene = new THREE.Scene(); + // scene.background = new THREE.Color(0x00ff00); + const height = 1000; + const width = 1000; + const camera = new THREE.OrthographicCamera( + width / -2, + width / 2, + height / 2, + height / -2 + ); + camera.position.set(0, 0, 2000); + camera.lookAt(new THREE.Vector3(0, 0, 0)); + const renderer = new THREE.WebGLRenderer({ + alpha: true, + antialias: true, + preserveDrawingBuffer: true, + }); + renderer.setSize(width, height); + + // --------------- Start of sprite stuff ---------------- + // const texture = new THREE.TextureLoader().load('src/images/dot.png'); + // const material = new THREE.PointsMaterial({ + // size: 5, + // vertexColors: true, + // map: texture, + // transparent: true, + // depthWrite: false, + // blending: THREE.NoBlending, + // }); + + // --------------- End of sprite stuff ---------------- + + // --------------- Start of shaders stuff ---------------- + const uniforms = { + pointTexture: { + value: new THREE.TextureLoader().load('src/images/dot.png'), + }, + }; + + const vertexShader = ` + attribute float size; + varying vec3 vColor; + void main() { + vColor = color; + vec4 mvPosition = modelViewMatrix * vec4( position, 1.0 ); + gl_PointSize = size; + gl_Position = projectionMatrix * mvPosition; + + }`; + const fragmentShader = ` + uniform sampler2D pointTexture; + varying vec3 vColor; + void main() { + gl_FragColor = vec4( vColor, 1.0 ); + gl_FragColor = gl_FragColor * texture2D( pointTexture, gl_PointCoord ); + }`; + + const material = new THREE.ShaderMaterial({ + uniforms: uniforms, + vertexShader: vertexShader, + fragmentShader: fragmentShader, + blending: THREE.AdditiveBlending, + depthTest: false, + transparent: true, + vertexColors: true, + }); + + // --------------- End of shaders stuff ---------------- + + const geometry = new THREE.BufferGeometry(); + + const controls = new OrbitControls(camera, renderer.domElement); + controls.mouseButtons = { + LEFT: THREE.MOUSE.PAN, + MIDDLE: THREE.MOUSE.MIDDLE, + RIGHT: THREE.MOUSE.RIGHT, + }; + controls.enableRotate = false; + + const result = { + renderer, + scene, + camera, + geometry, + material, + }; + + controls.addEventListener('change', () => renderScene(result)); + + return result; +}; + +const renderScene = ({ renderer, scene, camera }) => { + if (typeof renderer !== 'undefined') { + renderer.render(scene, camera); + } +}; + +const intitializeDataPoints = (scene, camera, geometry, material, coords) => { + const x = [] as any; + const y = [] as any; + const positions = [] as any; + const colors = [] as any; + const sizes = [] as any; + + // const multiplyCells = 50; + // const nRows = 10; + // const spacing = 30 + // let col = 0; + + // for (let n = 0; n < multiplyCells; ++n) { + // if (n % nRows == 0) { + // col += 1; + // } + // coords.forEach((coord: { x: number; y: number; }) => { + // x.push(coord.x + spacing * (n % nRows)); + // y.push(coord.y + spacing * col); + // positions.push(coord.x + spacing * (n % nRows), coord.y + spacing * col, 0); + // colors.push(255, 0, 0); + // sizes.push(20); + // }) + // } + coords.forEach((coord: { x: number; y: number }) => { + x.push(coord.x); + y.push(coord.y); + positions.push(coord.x, coord.y, 0); + colors.push(255, 0, 0); + sizes.push(5); + }); + + console.log( + 'Initializing data points...' + colors.length + ' ' + positions.length + ); + scene.clear(); + + const sorted_x = R.sort(R.subtract, x); + const sorted_y = R.sort(R.subtract, y); + + const [xMin, xMax] = [ + R.head(sorted_x) || 0, + R.nth(sorted_x.length - 1, sorted_x) || 0, + ]; + const [yMin, yMax] = [ + R.head(sorted_y) || 0, + R.nth(sorted_y.length - 1, sorted_y) || 0, + ]; + + const xCenter = (xMax + xMin) / 2; + const yCenter = (yMax + yMin) / 2; + + const maxDiff = + Math.max(Math.abs(xMax - xCenter), Math.abs(yMax - yCenter)) * 1.5; + const aspectRatio = 1; + camera.left = xCenter - maxDiff; + camera.right = xCenter + maxDiff; + camera.top = yCenter + maxDiff / aspectRatio; + camera.bottom = yCenter - maxDiff / aspectRatio; + + camera.updateProjectionMatrix(); + + geometry.setAttribute( + 'position', + new THREE.Float32BufferAttribute(positions, 3) + ); + geometry.setAttribute('color', new THREE.Float32BufferAttribute(colors, 3)); + geometry.setAttribute( + 'size', + new THREE.Float32BufferAttribute(sizes, 1).setUsage( + THREE.DynamicDrawUsage + ) + ); + + const points = new THREE.Points(geometry, material); + // const points = new THREE.Points(geometry); + scene.add(points); +}; + +export const Viewer: React.FC = (props: ViewerProps) => { + const dispatch = useDispatch(); + const coords = useSelector>( + (root: RootState) => { + return root.main.coords; + } + ); + const mount = useRef(null!); // nosemgrep: typescript.react.security.audit.react-no-refs.react-no-refs + const [rendererState, setRenderer] = React.useState(); + const [mounted, setMounted] = React.useState(false); + const [sceneState, setScene] = React.useState(); + const [cameraState, setCamera] = React.useState(); + const [geometryState, setGeometry] = React.useState(); + const [materialState, setMaterial] = React.useState(); + + const requestRef = useRef() as React.MutableRefObject; // nosemgrep: typescript.react.security.audit.react-no-refs.react-no-refs + const previousTimeRef = useRef(); // nosemgrep: typescript.react.security.audit.react-no-refs.react-no-refs + + const animate = (time) => { + if (previousTimeRef.current !== undefined) { + renderScene({ + renderer: rendererState, + scene: sceneState, + camera: cameraState, + }); + } + previousTimeRef.current = time; + requestRef.current = requestAnimationFrame(animate); + }; + + useEffect(() => { + const { renderer, scene, camera, geometry, material } = initGraphics(); + setRenderer(renderer); + setScene(scene); + setCamera(camera); + setGeometry(geometry); + setMaterial(material); + + if (!mounted) { + mount.current.appendChild(renderer.domElement); + setMounted(true); + } + + if (coords.length === 0) { + dispatch(getCoordinates(props.project, props.dataset)); + } + if (sceneState && cameraState && geometryState) { + intitializeDataPoints( + sceneState, + cameraState, + geometryState, + materialState, + coords + ); + } + + requestRef.current = requestAnimationFrame(animate); + return () => cancelAnimationFrame(requestRef.current); + }, [coords]); + + return
; // nosemgrep: typescript.react.security.audit.react-no-refs.react-no-refs +}; diff --git a/client/src/components/Viewer/ViewerWrapper.tsx b/client/src/components/Viewer/ViewerWrapper.tsx index 748cfd6a..a4c453b6 100644 --- a/client/src/components/Viewer/ViewerWrapper.tsx +++ b/client/src/components/Viewer/ViewerWrapper.tsx @@ -6,17 +6,10 @@ import { Button, Icon } from 'semantic-ui-react'; import * as MainSelect from '../../redux/selectors'; import { RootState } from '../../redux/reducers'; -import { ViewerId, ViewerInfo, ViewerMap } from './model'; +import { ViewerId, ViewerMap } from './model'; import * as Select from './selectors'; import * as Action from './actions'; - -const Placeholder: React.FC = (props: ViewerInfo) => { - return ( -
- {props.project}, {props.dataset} -
- ); -}; +import { Viewer } from './Viewer'; type WrapperState = { viewers: ViewerMap; @@ -68,9 +61,9 @@ export const ViewerWrapper: React.FC<{}> = () => { return
Empty
; } else { return ( - ); } diff --git a/client/src/redux/actionTypes.ts b/client/src/redux/actionTypes.ts index 60dd3c08..973d7306 100644 --- a/client/src/redux/actionTypes.ts +++ b/client/src/redux/actionTypes.ts @@ -21,3 +21,7 @@ export const UPLOAD_PROGRESS = 'UPLOAD_PROGRESS'; export const UPLOAD_SUCCESS = 'UPLOAD_SUCCESS'; export const MODIFIER_KEY_TOGGLE = 'MODIFIER_KEY_TOGGLE'; + +export const GET_COORDINATES = 'GET_COORDINATES'; + +export const RECEIVED_COORDINATES = 'RECEIVED_COORDINATES'; diff --git a/client/src/redux/actions.ts b/client/src/redux/actions.ts index b1617fc4..a09bd91f 100644 --- a/client/src/redux/actions.ts +++ b/client/src/redux/actions.ts @@ -1,4 +1,4 @@ -import { Project, DataSet } from '../api'; +import { Project, DataSet, Coordinate } from '../api'; import * as AT from './actionTypes'; import { @@ -10,6 +10,8 @@ import { AddDataSetAction, ModifierKey, ToggleModifierKey, + GetCoordinates, + ReceivedCoordinates, } from './types'; export const setAppLoading = (isAppLoading: MainState['isAppLoading']) => ({ @@ -74,3 +76,18 @@ export const toggleModifierKey = (key: ModifierKey): ToggleModifierKey => ({ type: AT.MODIFIER_KEY_TOGGLE, payload: { key }, }); + +export const getCoordinates = ( + project: string, + dataset: number +): GetCoordinates => ({ + type: AT.GET_COORDINATES, + payload: { project, dataset }, +}); + +export const receivedCoordinates = ( + coords: Array +): ReceivedCoordinates => ({ + type: AT.RECEIVED_COORDINATES, + payload: { coordinates: coords }, +}); diff --git a/client/src/redux/reducers/main.ts b/client/src/redux/reducers/main.ts index 8bec9ed1..0eddc9e9 100644 --- a/client/src/redux/reducers/main.ts +++ b/client/src/redux/reducers/main.ts @@ -9,6 +9,7 @@ const initialState: MainState = { sessionMode: SESSION_READ, projects: [], datasets: [], + coords: [], upload: { state: 'none', progress: 0, @@ -59,6 +60,10 @@ const main = produce((draft: MainState, action: MainAction) => { ? 'None' : action.payload.key; break; + + case Action.RECEIVED_COORDINATES: + draft.coords = action.payload.coordinates; + break; } }, initialState); diff --git a/client/src/redux/sagas/index.ts b/client/src/redux/sagas/index.ts index 32cda90a..0d51e65f 100644 --- a/client/src/redux/sagas/index.ts +++ b/client/src/redux/sagas/index.ts @@ -13,5 +13,6 @@ export default function* rootSaga() { SCOPE.watchPermalinkRequests(), SCOPE.watchCreateNewProject(), SCOPE.watchUploadRequest(), + SCOPE.watchGetCoordinates(), ]); } diff --git a/client/src/redux/sagas/scope/index.ts b/client/src/redux/sagas/scope/index.ts index c8cf6e34..1a140878 100644 --- a/client/src/redux/sagas/scope/index.ts +++ b/client/src/redux/sagas/scope/index.ts @@ -93,6 +93,19 @@ export function* watchUploadRequest() { yield takeEvery(AT.UPLOAD_REQUEST, uploadRequest); } +function* requestCoords(action: T.GetCoordinates) { + const coords = yield call( + API.getCoordinates, + action.payload.project, + action.payload.dataset + ); + yield put(A.receivedCoordinates(coords)); +} + +export function* watchGetCoordinates() { + yield takeEvery(AT.GET_COORDINATES, requestCoords); +} + export * from '../../../components/Search/effects'; export { watchGuestLogin, diff --git a/client/src/redux/types.ts b/client/src/redux/types.ts index 10ec5906..43e184b5 100644 --- a/client/src/redux/types.ts +++ b/client/src/redux/types.ts @@ -1,4 +1,4 @@ -import { Project, DataSet } from '../api'; +import { Project, DataSet, Coordinate } from '../api'; import * as AT from './actionTypes'; @@ -17,6 +17,7 @@ export interface MainState { sessionMode: SessionMode; projects: Array; datasets: Array; + coords: Array; upload: { state: UploadState; progress: number; @@ -106,6 +107,21 @@ export interface ToggleModifierKey { }; } +export interface GetCoordinates { + type: typeof AT.GET_COORDINATES; + payload: { + project: string; + dataset: number; + }; +} + +export interface ReceivedCoordinates { + type: typeof AT.RECEIVED_COORDINATES; + payload: { + coordinates: Array; + }; +} + export type MainAction = | SetLoadingAction | SetUUIDAction @@ -118,4 +134,6 @@ export type MainAction = | UploadRequest | UploadProgress | UploadSuccess - | ToggleModifierKey; + | ToggleModifierKey + | GetCoordinates + | ReceivedCoordinates; diff --git a/server/scopeserver/api/v1/__init__.py b/server/scopeserver/api/v1/__init__.py index e4669b72..3c0d81a8 100644 --- a/server/scopeserver/api/v1/__init__.py +++ b/server/scopeserver/api/v1/__init__.py @@ -2,10 +2,11 @@ from fastapi import APIRouter -from scopeserver.api.v1 import auth, projects, users, legacy +from scopeserver.api.v1 import auth, projects, users, legacy, data api_v1_router = APIRouter() api_v1_router.include_router(projects.router, prefix="/project", tags=["projects"]) api_v1_router.include_router(users.router, prefix="/user", tags=["users"]) api_v1_router.include_router(auth.router, prefix="/auth", tags=["auth"]) api_v1_router.include_router(legacy.router, prefix="/legacy", tags=["legacy"]) +api_v1_router.include_router(data.router, prefix="/data") diff --git a/server/scopeserver/api/v1/data.py b/server/scopeserver/api/v1/data.py new file mode 100644 index 00000000..f79ff98a --- /dev/null +++ b/server/scopeserver/api/v1/data.py @@ -0,0 +1,34 @@ +" TODO: THIS IS A TEMPORARY HACK JOB " + +from typing import List +from pathlib import Path + +from fastapi import APIRouter, Depends +from sqlalchemy.orm import Session + +import loompy as lp + +from scopeserver import crud, schemas +from scopeserver.api import deps +from scopeserver.config import settings + +router = APIRouter() + + +@router.get("/dataset", response_model=List[schemas.Coordinate]) +def get_dataset( + *, + database: Session = Depends(deps.get_db), + project: str, + dataset: int, +): + entry = crud.get_dataset(database, dataset) + + if entry is not None: + with lp.connect(settings.DATA_PATH / Path(project) / Path(entry.filename), validate=False) as ds: + xs = [_x[0] for _x in ds.ca.Embeddings_X] + ys = [_y[0] for _y in ds.ca.Embeddings_Y] + + return [schemas.Coordinate(x=X, y=Y) for X, Y in zip(xs, ys)] + + return [] diff --git a/server/scopeserver/schemas.py b/server/scopeserver/schemas.py index 395679cb..6f33cb97 100644 --- a/server/scopeserver/schemas.py +++ b/server/scopeserver/schemas.py @@ -23,6 +23,18 @@ class Config: orm_mode = True +class DatasetHack(Dataset): + project: "Project" + + class Config: + orm_mode = True + + +class Coordinate(BaseModel): + x: float + y: float + + # Users and Projects class UserBase(BaseModel): name: Optional[str]