diff --git a/data/speaker.csv b/data/speaker.csv new file mode 100644 index 0000000..b6f40fc --- /dev/null +++ b/data/speaker.csv @@ -0,0 +1,1032 @@ +id,sex,race +2852,F,CHINESE +2670,F,MALAY +3064,F,CHINESE +2862,F,CHINESE +2861,F,CHINESE +2857,M,CHINESE +2828,F,CHINESE +2858,F,CHINESE +2888,M,CHINESE +3159,M,INDIAN +3162,F,INDIAN +2954,F,CHINESE +2957,F,CHINESE +2952,M,CHINESE +2939,F,CHINESE +2945,F,CHINESE +2940,F,CHINESE +2953,M,CHINESE +2955,F,INDIAN +2946,M,CHINESE +2958,F,CHINESE +2968,F,CHINESE +2964,M,CHINESE +2959,M,CHINESE +2965,M,CHINESE +2947,F,CHINESE +2941,F,CHINESE +2960,F,CHINESE +2969,M,CHINESE +2974,F,CHINESE +2971,F,CHINESE +2942,F,CHINESE +2970,F,CHINESE +2961,F,CHINESE +2966,M,CHINESE +2948,M,CHINESE +2972,M,CHINESE +2962,M,CHINESE +2980,M,CHINESE +2967,F,INDIAN +2975,F,CHINESE +2987,F,CHINESE +2973,F,CHINESE +2976,F,CHINESE +2963,F,CHINESE +2981,F,CHINESE +2985,M,CHINESE +2992,M,CHINESE +2988,F,MALAY +2982,F,CHINESE +2977,F,CHINESE +2978,F,CHINESE +2983,M,CHINESE +2989,M,CHINESE +2990,M,MALAY +2979,F,MALAY +2984,F,CHINESE +2993,M,CHINESE +3009,F,INDIAN +3004,F,MALAY +2986,M,MALAY +3010,M,MALAY +2996,F,CHINESE +3005,F,CHINESE +2994,M,INDIAN +2991,F,INDIAN +3011,F,INDIAN +3006,M,MALAY +3012,F,MALAY +3013,M,MALAY +2998,F,MALAY +3001,M,MALAY +3015,M,MALAY +2997,F,MALAY +2995,F,MALAY +3014,F,MALAY +3007,F,CHINESE +2999,F,MALAY +3002,F,MALAY +3024,M,MALAY +3023,M,MALAY +3016,F,INDIAN +3017,M,MALAY +2877,F,CHINESE +2878,M,CHINESE +2879,F,INDIAN +3018,F,MALAY +3000,F,MALAY +3025,F,INDIAN +3020,M,MALAY +3008,F,CHINESE +3003,F,CHINESE +3019,F,INDIAN +3021,M,INDIAN +3026,M,MALAY +3041,F,INDIAN +3042,F,INDIAN +3033,F,CHINESE +3028,F,MALAY +3034,M,MALAY +3027,F,MALAY +3043,M,MALAY +3045,M,MALAY +3035,F,MALAY +3044,F,INDIAN +3029,M,INDIAN +3022,F,CHINESE +3050,F,CHINESE +3030,M,INDIAN +3081,M,CHINESE +3031,F,MALAY +3052,F,MALAY +3051,F,INDIAN +3062,M,INDIAN +3056,F,INDIAN +3057,F,MALAY +3046,F,MALAY +3048,M,MALAY +3054,F,CHINESE +3059,F,CHINESE +3058,F,CHINESE +3047,F,INDIAN +3053,F,INDIAN +3063,M,INDIAN +3032,M,MALAY +3065,M,CHINESE +3049,F,MALAY +3060,M,MALAY +3055,F,INDIAN +3070,M,INDIAN +3076,F,MALAY +3068,M,INDIAN +3061,M,MALAY +3066,M,MALAY +3071,M,INDIAN +3083,F,MALAY +3069,M,MALAY +3077,M,MALAY +3067,M,MALAY +3082,F,INDIAN +3093,M,CHINESE +3072,F,INDIAN +3078,F,INDIAN +3088,M,MALAY +3084,F,MALAY +3094,F,MALAY +3086,M,MALAY +3080,M,MALAY +3074,M,MALAY +3079,M,MALAY +3089,M,CHINESE +3085,M,CHINESE +3073,M,CHINESE +3099,F,INDIAN +3087,F,CHINESE +3075,M,CHINESE +3090,M,CHINESE +3095,M,INDIAN +3104,F,MALAY +3091,M,INDIAN +3100,M,CHINESE +3105,M,INDIAN +3109,M,INDIAN +3096,F,INDIAN +3092,M,CHINESE +3101,M,CHINESE +3106,M,CHINESE +3114,M,INDIAN +3097,M,MALAY +3103,M,CHINESE +3107,F,CHINESE +3110,M,INDIAN +3124,M,INDIAN +3108,M,INDIAN +3111,F,INDIAN +3115,M,INDIAN +3098,M,INDIAN +3130,F,INDIAN +3116,M,INDIAN +3125,M,INDIAN +3112,F,INDIAN +3136,F,INDIAN +3117,M,CHINESE +3113,M,CHINESE +3126,M,INDIAN +3118,M,CHINESE +3155,F,INDIAN +3127,M,INDIAN +3144,M,CHINESE +3131,F,INDIAN +3128,M,INDIAN +3141,M,INDIAN +3129,F,INDIAN +3132,M,INDIAN +3145,M,CHINESE +3137,F,INDIAN +3138,F,INDIAN +3133,M,INDIAN +3146,M,INDIAN +3150,F,INDIAN +3142,M,INDIAN +3143,M,INDIAN +3139,M,CHINESE +3147,M,INDIAN +3134,M,INDIAN +3151,F,INDIAN +3153,M,CHINESE +3140,M,INDIAN +3154,M,INDIAN +3148,F,INDIAN +3157,M,CHINESE +3158,M,CHINESE +3149,F,OTHERS +3135,F,INDIAN +3152,M,CHINESE +2831,F,CHINESE +3156,M,INDIAN +3161,M,OTHERS +3163,F,INDIAN +2603,M,MALAY +2605,M,MALAY +2607,M,MALAY +2614,M,MALAY +2615,M,MALAY +2616,M,MALAY +3191,F,INDIAN +2613,F,INDIAN +2612,M,CHINESE +2618,M,MALAY +2622,F,CHINESE +2621,F,CHINESE +2623,F,CHINESE +2630,M,CHINESE +2636,M,CHINESE +2643,F,CHINESE +2649,M,CHINESE +2242,M,CHINESE +2263,F,CHINESE +2269,F,CHINESE +2100,F,CHINESE +2101,M,CHINESE +2099,M,CHINESE +2129,F,CHINESE +2079,F,CHINESE +2077,F,CHINESE +2103,F,CHINESE +2139,F,CHINESE +2203,F,CHINESE +2108,F,INDIAN +2107,M,MALAY +2134,F,CHINESE +2141,F,CHINESE +2193,F,CHINESE +2207,M,INDIAN +2205,F,CHINESE +2102,M,CHINESE +2075,M,INDIAN +2357,M,CHINESE +2036,F,CHINESE +2300,F,CHINESE +2182,F,CHINESE +2319,F,CHINESE +2315,M,CHINESE +2246,F,MALAY +2247,F,MALAY +2249,M,CHINESE +2248,F,MALAY +2244,M,MALAY +2243,M,MALAY +2245,M,MALAY +2295,M,CHINESE +2356,F,CHINESE +2360,M,MALAY +2349,M,CHINESE +2353,M,CHINESE +2352,M,CHINESE +2332,F,CHINESE +2309,M,CHINESE +2318,M,INDIAN +2339,M,INDIAN +2329,F,CHINESE +2330,F,CHINESE +2301,F,CHINESE +2294,F,CHINESE +2317,F,INDIAN +2302,F,CHINESE +2299,M,CHINESE +2316,M,CHINESE +2325,F,OTHERS +2334,F,INDIAN +2326,M,CHINESE +2327,M,CHINESE +2328,M,CHINESE +2340,F,CHINESE +2361,M,MALAY +2358,M,CHINESE +2359,F,CHINESE +2369,F,CHINESE +2366,F,CHINESE +2400,M,INDIAN +2407,M,INDIAN +2419,M,MALAY +2424,M,MALAY +2425,M,MALAY +2422,M,INDIAN +2427,M,INDIAN +2428,M,MALAY +2430,M,INDIAN +2435,M,MALAY +2443,M,INDIAN +2442,M,MALAY +2441,M,MALAY +2468,M,INDIAN +2469,M,MALAY +2476,M,MALAY +2485,M,INDIAN +2487,M,INDIAN +2512,M,INDIAN +2513,M,INDIAN +2272,M,MALAY +2177,M,CHINESE +2232,M,MALAY +2114,F,CHINESE +2268,M,CHINESE +2221,M,CHINESE +2104,M,CHINESE +2210,F,INDIAN +2293,F,CHINESE +2304,F,CHINESE +2296,F,CHINESE +2306,F,CHINESE +2297,F,CHINESE +2307,F,CHINESE +2308,M,CHINESE +2298,M,CHINESE +2305,F,CHINESE +2310,M,CHINESE +2321,M,CHINESE +2324,M,CHINESE +2323,F,CHINESE +2322,F,CHINESE +2331,F,CHINESE +2336,F,CHINESE +2337,F,CHINESE +2338,F,CHINESE +2342,M,CHINESE +2341,F,CHINESE +2333,F,CHINESE +2344,F,CHINESE +2335,F,CHINESE +2346,F,CHINESE +2345,F,CHINESE +2354,M,CHINESE +2350,M,CHINESE +2351,F,CHINESE +2355,M,CHINESE +2343,M,CHINESE +2362,F,CHINESE +2363,F,CHINESE +2364,M,CHINESE +2365,M,CHINESE +2367,F,CHINESE +2370,F,CHINESE +2371,M,CHINESE +2372,F,MALAY +2373,F,MALAY +2374,F,CHINESE +2375,F,CHINESE +2376,F,CHINESE +2377,M,CHINESE +2384,F,CHINESE +2385,M,CHINESE +2386,F,CHINESE +2387,M,CHINESE +2388,F,CHINESE +2378,F,CHINESE +2379,F,CHINESE +2383,M,CHINESE +2382,F,CHINESE +2381,F,CHINESE +2380,M,CHINESE +2390,F,CHINESE +2391,M,CHINESE +2392,F,CHINESE +2393,F,CHINESE +2395,M,CHINESE +2396,M,CHINESE +2397,M,CHINESE +2398,M,CHINESE +2401,M,CHINESE +2402,M,INDIAN +2403,M,MALAY +2404,M,MALAY +2405,M,MALAY +2406,M,MALAY +2408,M,MALAY +2409,M,MALAY +2399,M,MALAY +2411,M,MALAY +2412,M,MALAY +2413,M,MALAY +2414,M,MALAY +2415,M,INDIAN +2416,M,MALAY +2417,M,MALAY +2420,M,INDIAN +2418,M,INDIAN +2421,M,MALAY +2423,M,MALAY +2426,M,MALAY +2432,M,MALAY +2434,M,MALAY +2436,M,MALAY +2437,M,MALAY +2438,M,MALAY +2439,M,MALAY +2440,M,INDIAN +2444,M,MALAY +2447,M,INDIAN +2448,M,MALAY +2453,M,MALAY +2455,M,MALAY +2454,M,MALAY +2456,M,MALAY +2458,M,MALAY +2459,M,INDIAN +2460,M,INDIAN +2461,M,INDIAN +2462,M,INDIAN +2463,M,INDIAN +2464,M,MALAY +2465,M,MALAY +2466,M,INDIAN +2471,F,INDIAN +2470,F,INDIAN +2467,M,INDIAN +2473,M,INDIAN +2474,M,INDIAN +2477,M,MALAY +2478,M,INDIAN +2479,F,INDIAN +2480,F,INDIAN +2481,M,INDIAN +2472,F,INDIAN +2484,M,INDIAN +2483,M,MALAY +2488,M,INDIAN +2489,M,INDIAN +2491,M,INDIAN +2492,M,INDIAN +2493,M,INDIAN +2494,M,INDIAN +2495,M,INDIAN +2496,M,CHINESE +2497,M,INDIAN +2498,M,MALAY +2499,M,INDIAN +2500,M,CHINESE +2502,M,CHINESE +2501,M,CHINESE +2503,M,CHINESE +2504,M,INDIAN +2505,M,INDIAN +2506,M,INDIAN +2507,M,INDIAN +2508,M,INDIAN +2509,M,INDIAN +2510,M,CHINESE +2511,M,CHINESE +2514,M,INDIAN +2516,M,INDIAN +2517,M,INDIAN +2518,M,CHINESE +2519,M,CHINESE +2520,M,CHINESE +2521,M,CHINESE +2522,M,CHINESE +2523,M,OTHERS +2524,M,CHINESE +2525,M,CHINESE +2526,M,CHINESE +2527,M,INDIAN +2530,M,CHINESE +2528,M,CHINESE +2531,M,CHINESE +2529,M,CHINESE +2532,M,MALAY +2533,M,MALAY +2534,M,MALAY +2535,M,CHINESE +2536,M,CHINESE +2537,M,CHINESE +2538,M,CHINESE +2539,M,CHINESE +2540,M,CHINESE +2541,M,CHINESE +2542,M,CHINESE +2543,M,CHINESE +2544,M,CHINESE +2545,M,CHINESE +2546,M,CHINESE +2547,M,CHINESE +2548,F,INDIAN +2389,M,INDIAN +2394,F,CHINESE +2486,M,INDIAN +2283,F,MALAY +2235,M,CHINESE +2482,F,INDIAN +2064,M,CHINESE +2078,M,CHINESE +2158,M,CHINESE +2190,F,CHINESE +2234,F,MALAY +2450,M,CHINESE +2452,M,CHINESE +2005,F,CHINESE +2061,M,CHINESE +2011,F,CHINESE +2016,F,CHINESE +2220,F,CHINESE +2122,F,CHINESE +2118,F,CHINESE +2119,M,CHINESE +2024,F,INDIAN +2022,M,CHINESE +2219,F,CHINESE +2001,F,MALAY +2002,F,INDIAN +2003,M,MALAY +2716,M,CHINESE +2723,F,CHINESE +2675,M,MALAY +2674,M,MALAY +2712,M,CHINESE +2720,M,CHINESE +2684,F,INDIAN +2711,M,MALAY +2699,F,CHINESE +2707,M,CHINESE +2722,F,CHINESE +2721,F,CHINESE +2732,F,CHINESE +2685,M,CHINESE +2738,F,CHINESE +2676,M,CHINESE +2733,F,CHINESE +2739,F,CHINESE +2724,F,CHINESE +2754,F,INDIAN +2744,F,CHINESE +2734,F,MALAY +2753,F,MALAY +2725,M,CHINESE +2740,M,CHINESE +2736,M,CHINESE +2749,F,CHINESE +2776,M,CHINESE +2750,F,CHINESE +2767,F,CHINESE +2772,F,CHINESE +2766,F,CHINESE +2745,F,CHINESE +2746,F,CHINESE +2735,M,CHINESE +2742,M,INDIAN +2741,M,CHINESE +2782,F,CHINESE +2773,F,CHINESE +2774,F,CHINESE +2726,F,CHINESE +2768,M,MALAY +2770,F,MALAY +2743,F,CHINESE +2788,F,CHINESE +2780,M,CHINESE +2784,M,INDIAN +2775,F,CHINESE +2727,F,MALAY +2728,F,INDIAN +2789,F,CHINESE +2783,F,CHINESE +2792,F,INDIAN +2769,F,MALAY +2790,F,CHINESE +2798,M,CHINESE +2799,M,INDIAN +2779,F,CHINESE +2778,M,CHINESE +2785,F,INDIAN +2794,F,CHINESE +2729,F,MALAY +2791,M,CHINESE +2793,F,INDIAN +2786,F,MALAY +2829,M,MALAY +2787,M,CHINESE +2781,M,MALAY +2795,M,MALAY +2800,F,MALAY +2835,F,CHINESE +2801,F,CHINESE +2830,F,CHINESE +2802,F,MALAY +2796,F,MALAY +2839,F,CHINESE +2832,F,INDIAN +2797,F,CHINESE +2836,M,MALAY +2837,F,INDIAN +2803,F,CHINESE +2755,M,CHINESE +2834,F,CHINESE +2846,F,CHINESE +2823,F,CHINESE +2840,F,CHINESE +2833,M,MALAY +2845,M,CHINESE +2841,F,CHINESE +2854,M,CHINESE +2824,F,CHINESE +2847,F,CHINESE +2838,F,CHINESE +2842,F,CHINESE +2825,M,CHINESE +2853,M,CHINESE +2843,M,CHINESE +2826,F,CHINESE +2850,F,CHINESE +2859,F,CHINESE +2855,M,CHINESE +2849,F,CHINESE +2848,F,CHINESE +2827,F,CHINESE +2851,M,CHINESE +2856,M,CHINESE +2860,F,CHINESE +2804,F,CHINESE +2805,F,CHINESE +2869,F,CHINESE +2866,F,CHINESE +2806,F,CHINESE +2870,F,CHINESE +2811,M,CHINESE +2864,F,CHINESE +2810,M,CHINESE +2865,F,CHINESE +2863,F,CHINESE +2883,F,CHINESE +2812,M,CHINESE +2871,M,CHINESE +2867,F,CHINESE +2807,F,CHINESE +2885,F,CHINESE +2809,M,CHINESE +2894,F,CHINESE +2890,F,CHINESE +2808,F,CHINESE +2872,M,CHINESE +2884,M,CHINESE +2868,F,CHINESE +2901,F,CHINESE +2891,M,CHINESE +2895,M,CHINESE +2886,F,CHINESE +2896,F,CHINESE +2887,M,CHINESE +2913,M,CHINESE +2893,M,CHINESE +2905,F,CHINESE +2892,F,CHINESE +2904,M,CHINESE +2898,M,CHINESE +2899,M,CHINESE +2914,F,CHINESE +2909,F,CHINESE +2915,M,INDIAN +2910,M,CHINESE +2924,M,INDIAN +2906,M,MALAY +2918,F,CHINESE +2925,F,CHINESE +2919,M,CHINESE +2911,M,CHINESE +2916,F,CHINESE +2908,M,CHINESE +2912,F,CHINESE +2917,F,CHINESE +2926,M,CHINESE +2920,M,CHINESE +2907,M,CHINESE +2921,M,CHINESE +2930,F,CHINESE +2937,F,CHINESE +2943,F,CHINESE +2931,F,CHINESE +2938,F,CHINESE +2928,F,CHINESE +2927,F,CHINESE +2922,F,CHINESE +2932,F,CHINESE +2933,F,CHINESE +2929,M,MALAY +2923,M,CHINESE +2934,F,INDIAN +2936,F,CHINESE +2944,M,CHINESE +2935,F,CHINESE +2949,F,CHINESE +2956,M,CHINESE +2620,M,CHINESE +3500,F,CHINESE +2664,M,OTHERS +2758,M,CHINESE +2759,M,CHINESE +2757,M,CHINESE +2761,F,CHINESE +2760,M,CHINESE +2762,M,CHINESE +2813,M,CHINESE +2815,F,MALAY +2814,F,MALAY +2817,M,CHINESE +2818,M,INDIAN +2819,M,OTHERS +2820,F,OTHERS +2821,F,INDIAN +2873,F,MALAY +2874,F,INDIAN +2876,F,OTHERS +2756,F,CHINESE +2844,F,CHINESE +2638,M,INDIAN +2601,M,CHINESE +2602,F,CHINESE +2604,F,CHINESE +2606,F,CHINESE +2608,F,CHINESE +2611,F,CHINESE +2624,F,CHINESE +2625,F,CHINESE +2626,F,CHINESE +2627,M,CHINESE +2628,F,CHINESE +2629,M,CHINESE +2631,F,CHINESE +2632,F,CHINESE +2619,F,CHINESE +2634,F,CHINESE +2695,M,CHINESE +2637,F,CHINESE +2641,M,MALAY +2656,F,CHINESE +2658,F,MALAY +2659,F,CHINESE +2660,M,CHINESE +2665,F,CHINESE +2661,F,CHINESE +2662,M,CHINESE +2666,F,CHINESE +2667,M,MALAY +2669,M,CHINESE +2671,F,MALAY +2672,F,CHINESE +2673,F,CHINESE +2681,F,CHINESE +2683,F,MALAY +2688,F,CHINESE +2687,F,CHINESE +2691,M,CHINESE +2694,F,CHINESE +2655,M,CHINESE +2692,M,CHINESE +2693,F,CHINESE +2639,M,CHINESE +2696,M,CHINESE +2701,M,CHINESE +2697,M,MALAY +2698,F,MALAY +2702,M,CHINESE +2703,M,MALAY +2704,F,MALAY +2705,M,CHINESE +2706,M,MALAY +2690,M,CHINESE +2708,F,MALAY +2709,F,CHINESE +2710,F,MALAY +2713,F,MALAY +2714,F,CHINESE +2715,F,CHINESE +2663,M,CHINESE +2635,M,CHINESE +2644,M,INDIAN +2679,F,INDIAN +2640,M,MALAY +2642,M,INDIAN +2645,M,CHINESE +2654,F,MALAY +2668,F,MALAY +2680,F,INDIAN +2609,F,CHINESE +2610,F,CHINESE +2647,M,INDIAN +2155,M,CHINESE +2062,M,CHINESE +2123,F,CHINESE +2020,M,CHINESE +2113,F,CHINESE +2017,F,CHINESE +2023,M,CHINESE +2266,M,CHINESE +2273,M,CHINESE +2311,F,CHINESE +2025,M,CHINESE +2066,M,CHINESE +2281,F,CHINESE +2270,F,CHINESE +2124,F,CHINESE +2069,F,CHINESE +2028,F,CHINESE +2121,F,CHINESE +2169,F,CHINESE +2083,F,CHINESE +2084,M,CHINESE +2082,F,CHINESE +2167,F,MALAY +2267,F,CHINESE +2120,F,CHINESE +2049,F,CHINESE +2106,F,INDIAN +2171,F,CHINESE +2070,F,CHINESE +2170,M,CHINESE +2222,F,CHINESE +2117,M,INDIAN +2030,F,Chinese +2040,F,Chinese +2174,F,CHINESE +2035,M,CHINESE +2175,F,CHINESE +2176,F,CHINESE +2179,M,CHINESE +2071,F,CHINESE +2223,F,CHINESE +2224,M,CHINESE +2038,M,MALAY +2226,F,CHINESE +2225,F,CHINESE +2274,M,CHINESE +2034,F,CHINESE +2095,F,CHINESE +2094,F,CHINESE +2228,M,MALAY +2029,M,CHINESE +2230,F,CHINESE +2043,F,CHINESE +2178,F,MALAY +2202,F,CHINESE +2180,M,CHINESE +2181,F,MALAY +2046,F,CHINESE +2184,F,CHINESE +2092,F,CHINESE +2227,F,MALAY +2048,M,CHINESE +2183,F,CHINESE +2109,M,MALAY +2275,M,CHINESE +2045,F,CHINESE +2047,M,CHINESE +2277,F,CHINESE +2231,F,INDIAN +2278,F,MALAY +2279,F,CHINESE +2052,F,CHINESE +2073,F,INDIAN +2280,M,CHINESE +2041,F,CHINESE +2042,M,CHINESE +2051,M,CHINESE +2059,F,MALAY +2133,M,MALAY +2060,M,CHINESE +2132,F,MALAY +2131,M,MALAY +2072,F,CHINESE +2033,M,CHINESE +2189,M,MALAY +2039,F,MALAY +2105,F,CHINESE +2130,F,MALAY +2191,F,INDIAN +2074,M,INDIAN +2058,M,CHINESE +2312,M,INDIAN +2053,F,CHINESE +2050,M,INDIAN +2097,F,CHINESE +2044,M,MALAY +2057,M,INDIAN +2056,F,INDIAN +2055,M,INDIAN +2054,M,INDIAN +2081,F,CHINESE +2204,F,CHINESE +2076,F,CHINESE +2125,F,CHINESE +2136,F,INDIAN +2080,M,CHINESE +2137,F,INDIAN +2135,F,INDIAN +2088,F,MALAY +2086,F,MALAY +2091,F,MALAY +2090,F,INDIAN +2085,F,CHINESE +2093,F,INDIAN +2110,M,INDIAN +2087,F,MALAY +2089,F,CHINESE +2098,F,MALAY +2150,M,CHINESE +2147,F,INDIAN +2148,F,INDIAN +2149,F,INDIAN +2145,M,MALAY +2138,F,CHINESE +2144,F,CHINESE +2128,F,CHINESE +2156,F,MALAY +2157,M,INDIAN +2153,M,INDIAN +2146,F,CHINESE +2127,F,CHINESE +2198,F,MALAY +2143,F,INDIAN +2142,M,MALAY +2140,M,CHINESE +2151,F,CHINESE +2200,M,CHINESE +2154,M,INDIAN +2152,F,CHINESE +2185,F,CHINESE +2186,F,INDIAN +2194,M,CHINESE +2159,M,CHINESE +2187,F,CHINESE +2196,M,CHINESE +2188,M,CHINESE +2208,M,INDIAN +2160,M,INDIAN +2192,M,CHINESE +2199,M,CHINESE +2201,M,OTHERS +2206,M,MALAY +2037,M,MALAY +2096,M,CHINESE +2172,F,CHINESE +2195,M,CHINESE +2197,M,MALAY +2233,M,CHINESE +2271,M,CHINESE +2257,F,CHINESE +2251,F,INDIAN +2284,M,CHINESE +2254,F,CHINESE +2250,F,INDIAN +2238,M,CHINESE +2253,M,CHINESE +2209,M,CHINESE +2252,F,CHINESE +2259,M,CHINESE +2239,F,CHINESE +2241,M,CHINESE +2240,F,CHINESE +2237,F,CHINESE +2258,F,CHINESE +2255,M,INDIAN +2282,F,MALAY +2260,F,MALAY +2285,F,CHINESE +2287,F,CHINESE +2288,F,INDIAN +2289,F,CHINESE +2292,F,CHINESE +2286,M,INDIAN +2236,M,CHINESE +2290,F,CHINESE +2291,F,INDIAN +2008,M,CHINESE +2009,M,CHINESE +2007,M,CHINESE +2012,M,CHINESE +2018,M,CHINESE +2111,M,CHINESE +2013,M,MALAY +2006,M,CHINESE +2010,F,CHINESE +2112,F,CHINESE +2014,M,CHINESE +2115,F,CHINESE +2015,M,CHINESE +2261,F,CHINESE +2116,M,CHINESE +2161,F,CHINESE +2211,M,MALAY +2262,F,MALAY +2264,F,CHINESE +2212,M,CHINESE +2162,F,CHINESE +2163,M,CHINESE +2213,F,CHINESE +2265,F,CHINESE +2164,M,CHINESE +2021,F,CHINESE +2063,M,CHINESE +2214,F,CHINESE +2165,F,CHINESE +2215,M,CHINESE +2166,F,CHINESE +2217,M,MALAY +2218,F,CHINESE +2027,F,CHINESE +2067,F,CHINESE +2065,F,INDIAN +2026,F,CHINESE +2068,M,CHINESE +2173,F,CHINESE +2216,M,CHINESE +2650,F,INDIAN diff --git a/data/speech_sample_output.csv b/data/speech_sample_output.csv new file mode 100644 index 0000000..2d65eac --- /dev/null +++ b/data/speech_sample_output.csv @@ -0,0 +1,299 @@ +sex,race,truth_count,match_count +F,CHINESE,77,60 +F,CHINESE,84,64 +M,CHINESE,80,53 +F,INDIAN,80,53 +M,CHINESE,70,45 +F,CHINESE,81,50 +M,CHINESE,78,56 +M,CHINESE,76,52 +F,CHINESE,76,50 +F,CHINESE,77,53 +M,MALAY,77,53 +F,CHINESE,82,46 +M,CHINESE,79,52 +F,INDIAN,82,59 +F,MALAY,85,50 +M,MALAY,72,55 +F,CHINESE,80,58 +F,INDIAN,79,56 +F,MALAY,74,46 +M,MALAY,75,52 +F,MALAY,70,48 +F,MALAY,77,48 +F,MALAY,84,52 +F,MALAY,75,53 +F,MALAY,75,51 +M,MALAY,85,48 +F,INDIAN,79,63 +F,INDIAN,82,64 +F,INDIAN,76,51 +M,MALAY,75,47 +F,INDIAN,81,54 +F,INDIAN,77,59 +F,MALAY,80,57 +M,MALAY,78,51 +F,MALAY,73,49 +F,INDIAN,81,60 +M,INDIAN,79,59 +F,INDIAN,86,66 +F,MALAY,92,60 +M,MALAY,80,59 +M,CHINESE,83,59 +M,MALAY,85,66 +F,INDIAN,83,64 +M,INDIAN,85,50 +M,INDIAN,80,57 +M,MALAY,81,55 +M,MALAY,86,51 +M,MALAY,86,55 +F,INDIAN,81,55 +M,MALAY,82,53 +M,MALAY,79,49 +M,MALAY,78,55 +M,MALAY,85,55 +F,MALAY,79,54 +M,INDIAN,81,52 +F,INDIAN,76,62 +M,CHINESE,78,52 +M,INDIAN,79,59 +M,INDIAN,80,52 +M,INDIAN,77,57 +F,INDIAN,89,59 +M,INDIAN,87,53 +M,CHINESE,90,66 +M,INDIAN,72,57 +F,INDIAN,76,54 +M,INDIAN,82,58 +M,CHINESE,77,56 +M,INDIAN,78,47 +M,INDIAN,75,50 +F,INDIAN,81,58 +F,INDIAN,80,60 +M,INDIAN,77,56 +M,INDIAN,80,60 +M,INDIAN,77,55 +M,CHINESE,79,54 +F,INDIAN,75,56 +M,INDIAN,78,60 +F,INDIAN,83,66 +M,MALAY,65,49 +M,MALAY,71,54 +F,INDIAN,70,45 +M,MALAY,67,56 +F,CHINESE,75,49 +F,CHINESE,84,55 +F,CHINESE,88,62 +F,CHINESE,88,54 +M,INDIAN,79,51 +M,CHINESE,80,51 +F,CHINESE,83,58 +F,MALAY,77,51 +F,MALAY,78,54 +F,MALAY,78,58 +M,MALAY,85,54 +M,MALAY,78,59 +F,CHINESE,79,50 +F,INDIAN,77,53 +M,CHINESE,75,56 +F,INDIAN,82,51 +M,CHINESE,86,59 +F,CHINESE,81,58 +M,MALAY,81,53 +M,INDIAN,78,47 +M,MALAY,77,43 +M,MALAY,74,51 +M,INDIAN,81,55 +M,INDIAN,86,48 +M,MALAY,81,51 +M,INDIAN,81,58 +M,MALAY,89,65 +M,INDIAN,81,55 +M,INDIAN,85,56 +M,INDIAN,97,78 +M,MALAY,76,50 +M,MALAY,74,57 +F,CHINESE,75,52 +F,CHINESE,82,56 +M,CHINESE,93,50 +F,CHINESE,96,67 +F,CHINESE,80,57 +F,CHINESE,80,52 +M,CHINESE,80,56 +F,MALAY,84,55 +M,CHINESE,78,49 +M,CHINESE,76,51 +M,CHINESE,85,58 +M,MALAY,78,56 +M,MALAY,81,56 +M,MALAY,74,55 +M,MALAY,78,49 +M,MALAY,73,49 +M,MALAY,84,61 +M,INDIAN,84,60 +M,MALAY,83,57 +M,MALAY,72,52 +M,MALAY,85,61 +M,MALAY,89,70 +M,INDIAN,85,67 +M,MALAY,89,56 +M,MALAY,80,58 +M,MALAY,81,59 +M,MALAY,82,52 +M,INDIAN,92,57 +M,INDIAN,88,60 +F,INDIAN,86,60 +M,INDIAN,86,63 +M,MALAY,92,67 +M,INDIAN,73,55 +F,INDIAN,84,53 +F,INDIAN,79,59 +M,INDIAN,83,52 +M,MALAY,85,58 +M,INDIAN,78,58 +M,INDIAN,86,60 +M,INDIAN,81,55 +M,CHINESE,78,50 +M,INDIAN,81,59 +M,INDIAN,81,42 +M,INDIAN,93,57 +M,INDIAN,86,58 +M,INDIAN,84,56 +M,INDIAN,84,61 +M,CHINESE,97,63 +M,INDIAN,81,57 +M,MALAY,77,45 +M,MALAY,80,58 +M,MALAY,98,62 +M,CHINESE,87,56 +F,INDIAN,74,50 +F,MALAY,82,56 +M,CHINESE,82,63 +F,CHINESE,79,60 +F,INDIAN,60,34 +M,MALAY,79,48 +F,INDIAN,78,50 +M,MALAY,77,58 +F,CHINESE,92,61 +F,MALAY,81,58 +F,CHINESE,87,52 +F,CHINESE,90,65 +M,INDIAN,93,60 +F,CHINESE,86,64 +F,CHINESE,84,57 +F,CHINESE,94,62 +M,INDIAN,88,69 +F,MALAY,79,60 +F,INDIAN,79,45 +F,MALAY,91,63 +M,INDIAN,86,62 +M,CHINESE,89,58 +F,INDIAN,89,69 +F,MALAY,76,48 +M,MALAY,108,61 +M,MALAY,88,67 +F,MALAY,77,57 +F,CHINESE,78,54 +F,MALAY,82,47 +M,MALAY,89,72 +F,CHINESE,104,55 +M,MALAY,96,64 +M,CHINESE,85,53 +F,CHINESE,93,60 +M,CHINESE,93,61 +F,CHINESE,85,49 +M,CHINESE,89,61 +F,CHINESE,78,43 +F,CHINESE,92,68 +F,CHINESE,84,60 +M,CHINESE,87,60 +M,CHINESE,78,45 +F,CHINESE,86,48 +F,CHINESE,88,53 +M,CHINESE,105,67 +M,MALAY,76,57 +M,CHINESE,72,56 +F,CHINESE,73,52 +F,INDIAN,82,43 +F,CHINESE,78,52 +M,CHINESE,94,56 +M,CHINESE,87,68 +M,CHINESE,90,59 +F,MALAY,81,57 +F,MALAY,88,63 +M,INDIAN,84,56 +F,INDIAN,104,54 +M,INDIAN,76,59 +F,CHINESE,70,49 +M,MALAY,77,47 +M,CHINESE,78,51 +F,CHINESE,67,51 +F,MALAY,74,50 +F,MALAY,77,51 +M,MALAY,71,50 +F,CHINESE,76,53 +M,MALAY,78,49 +M,CHINESE,69,46 +F,MALAY,65,40 +F,MALAY,71,49 +F,CHINESE,87,48 +M,INDIAN,73,47 +F,CHINESE,76,45 +M,CHINESE,80,55 +M,CHINESE,85,52 +F,CHINESE,74,56 +F,MALAY,87,62 +F,CHINESE,100,58 +F,INDIAN,86,57 +F,CHINESE,79,50 +M,INDIAN,87,49 +M,MALAY,72,49 +M,MALAY,83,56 +F,CHINESE,88,62 +F,CHINESE,76,61 +M,MALAY,94,73 +M,CHINESE,82,56 +F,INDIAN,87,61 +F,MALAY,85,52 +M,MALAY,90,59 +F,MALAY,81,61 +F,CHINESE,86,62 +F,MALAY,80,48 +F,MALAY,72,49 +M,INDIAN,77,54 +M,INDIAN,83,51 +M,INDIAN,87,60 +F,INDIAN,82,53 +M,INDIAN,83,54 +F,INDIAN,78,54 +F,MALAY,79,55 +F,MALAY,76,52 +F,INDIAN,78,59 +F,INDIAN,82,65 +M,MALAY,85,61 +M,INDIAN,87,59 +F,INDIAN,101,65 +F,CHINESE,77,46 +F,INDIAN,86,58 +M,CHINESE,89,63 +M,CHINESE,79,57 +M,INDIAN,83,61 +M,MALAY,72,56 +M,MALAY,89,57 +M,CHINESE,79,54 +F,INDIAN,81,55 +F,INDIAN,77,52 +M,CHINESE,83,51 +M,CHINESE,71,44 +F,CHINESE,74,53 +M,INDIAN,85,63 +F,MALAY,76,42 +F,CHINESE,78,49 +F,INDIAN,88,64 +F,INDIAN,80,56 +M,CHINESE,65,47 +M,CHINESE,74,50 +M,MALAY,68,52 +F,CHINESE,82,58 +F,CHINESE,79,52 +M,CHINESE,82,57 diff --git a/examples/get_output_from_google_speech_model.ipynb b/examples/get_output_from_google_speech_model.ipynb new file mode 100644 index 0000000..99d17dc --- /dev/null +++ b/examples/get_output_from_google_speech_model.ipynb @@ -0,0 +1,690 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Input .wav files from gs into google's speech to text model " + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "import sys \n", + "import re\n", + "\n", + "sys.path.insert(0, '..')\n", + "\n", + "# Add google app crendtials\n", + "#os.environ[\"GOOGLE_APPLICATION_CREDENTIALS\"]=\"\"\n", + "\n", + "pd.set_option('max_colwidth', 1000)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "# Read participants df\n", + "df=pd.read_csv(\"../data/speaker.csv\")\n", + "\n", + "# Sample 100 IDs each from each race\n", + "id_lst=list(pd.concat(\n", + " [\n", + " df[df.race=='CHINESE'].sample(100, random_state=99),\n", + " df[df.race=='MALAY'].sample(100, random_state=99),\n", + " df[df.race=='INDIAN'].sample(100, random_state=99)\n", + " ]\n", + ").id)\n", + "\n", + "# Remove 2 errors id\n", + "id_lst.remove(2888)\n", + "id_lst.remove(3151)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idsexrace
02852FCHINESE
12670FMALAY
23064FCHINESE
32862FCHINESE
42861FCHINESE
............
10262026FCHINESE
10272068MCHINESE
10282173FCHINESE
10292216MCHINESE
10302650FINDIAN
\n", + "

1031 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " id sex race\n", + "0 2852 F CHINESE\n", + "1 2670 F MALAY\n", + "2 3064 F CHINESE\n", + "3 2862 F CHINESE\n", + "4 2861 F CHINESE\n", + "... ... .. ...\n", + "1026 2026 F CHINESE\n", + "1027 2068 M CHINESE\n", + "1028 2173 F CHINESE\n", + "1029 2216 M CHINESE\n", + "1030 2650 F INDIAN\n", + "\n", + "[1031 rows x 3 columns]" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "# Imports the Google Cloud libraries\n", + "from google.cloud import speech\n", + "from google.cloud import storage\n", + "\n", + "\n", + "# Instantiates a client\n", + "client = speech.SpeechClient()\n", + "\n", + "# Get the first 10 text outputs of each sampled IDs using google speech model\n", + "for p in id_lst:\n", + " lst=[]\n", + " i=0\n", + " o=1\n", + " # Speech file numbers can be missing inbetween\n", + " while i!=10:\n", + " if o>300:\n", + " break\n", + " name = f\"out/0{p}{o:04d}.WAV\" \n", + " storage_client = storage.Client()\n", + " bucket_name = 'national_speech_corpus'\n", + " bucket = storage_client.bucket(bucket_name)\n", + " \n", + " # Check if file exists\n", + " if storage.Blob(bucket=bucket, name=name).exists(storage_client):\n", + " gcs_uri = f\"gs://national_speech_corpus/out/0{p}00{o:02d}.WAV\"\n", + " audio = speech.RecognitionAudio(uri=gcs_uri)\n", + " config = speech.RecognitionConfig(\n", + " language_code=\"en-SG\",\n", + " enable_automatic_punctuation=False,\n", + " )\n", + " # Detects speech in the audio file\n", + " response = client.recognize(config=config, audio=audio)\n", + " \n", + " # Check if there's output from the model\n", + " if len(response.results) > 0:\n", + " sentence = response.results[0].alternatives[0].transcript\n", + "\n", + " # Replace digits into words\n", + " my_dict = {'0': 'zero ', '1': 'one ', '2': 'two ', '3': 'three ', '4': 'four ', '5': 'five ', '6': 'six ', '7': 'seven ', '8': 'eight ', '9': 'nine '}\n", + " for item in sentence:\n", + " if item in my_dict.keys():\n", + " sentence=sentence.replace(item, my_dict[item])\n", + " lst.append(sentence)\n", + " else:\n", + " lst.append('')\n", + " i+=1\n", + " o+=1\n", + " \n", + " # Read transcript (truth) file for a particular participant\n", + " # Drop even index, first column and keep only first 10 sentences \n", + " output_df = pd.read_csv(f\"script/0{p}0.txt\", sep=\"\\t\", header=None).iloc[1:20:2].drop(0,axis=1).reset_index(drop=True)\n", + " output_df.columns=['truth']\n", + " output_df['output']=lst\n", + " output_df.to_csv(f\"output/{p}.csv\",index=False)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\Clare\\AppData\\Local\\Temp/ipykernel_6324/1510333972.py:3: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " df_speaker['truth_count']=0\n", + "C:\\Users\\Clare\\AppData\\Local\\Temp/ipykernel_6324/1510333972.py:4: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " df_speaker['match_count']=0\n", + "C:\\Users\\Clare\\AppData\\Roaming\\Python\\Python38\\site-packages\\pandas\\core\\indexing.py:1817: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " self._setitem_single_column(loc, value, pi)\n" + ] + } + ], + "source": [ + "# Add accuracy metrics to the output df and sampled speaker df\n", + "df_speaker=df[df.id.isin(id_lst)]\n", + "df_speaker['truth_count']=0\n", + "df_speaker['match_count']=0\n", + "for p in id_lst:\n", + " output_df = pd.read_csv(f'output/{p}.csv')\n", + " output_df['truth_split'] = output_df.truth.apply(lambda x: str(x).lower().split())\n", + " output_df['output_split'] = output_df.output.apply(lambda x: str(x).lower().split())\n", + " output_df['truth_count']=output_df.truth_split.apply(lambda x: len(set(x)))\n", + " output_df['match_count']=output_df.apply(lambda x:len(set(x.truth_split).intersection(set(x.output_split))),axis=1) \n", + " output_df.to_csv(f\"output/{p}.csv\",index=False)\n", + " \n", + " df_speaker.loc[df_speaker.id==p,'truth_count']=output_df.truth_count.sum()\n", + " df_speaker.loc[df_speaker.id==p,'match_count']=output_df.match_count.sum()\n", + "\n", + "df_speaker.reset_index().to_csv('speech_sample_output.csv',index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
truthoutputtruth_splitoutput_splittruth_countmatch_count
0ninenine['nine']['nine']11
1De Fabio Factory and Appledisable Factory in apple['de', 'fabio', 'factory', 'and', 'apple']['disable', 'factory', 'in', 'apple']52
2Oyakodon Yakizakana and Carrot Halwaoil Condon and carrot halwa['oyakodon', 'yakizakana', 'and', 'carrot', 'halwa']['oil', 'condon', 'and', 'carrot', 'halwa']53
3where can I find the best Red rubywhere can I find the best way to be['where', 'can', 'i', 'find', 'the', 'best', 'red', 'ruby']['where', 'can', 'i', 'find', 'the', 'best', 'way', 'to', 'be']86
4Kuih Kaswi waffle and Coconut KuihQuaker sweet waffle in Coconut Creek['kuih', 'kaswi', 'waffle', 'and', 'coconut', 'kuih']['quaker', 'sweet', 'waffle', 'in', 'coconut', 'creek']52
5please tell me how to cook Satay Babatplease tell me how to cook saute Bobbitt['please', 'tell', 'me', 'how', 'to', 'cook', 'satay', 'babat']['please', 'tell', 'me', 'how', 'to', 'cook', 'saute', 'bobbitt']86
6six five two three nine seven nine threesix five to three nine seven nine three['six', 'five', 'two', 'three', 'nine', 'seven', 'nine', 'three']['six', 'five', 'to', 'three', 'nine', 'seven', 'nine', 'three']65
7Denis D Cotta Paine Eric and Chong Tze ChienDennis Dakota and Chong searching['denis', 'd', 'cotta', 'paine', 'eric', 'and', 'chong', 'tze', 'chien']['dennis', 'dakota', 'and', 'chong', 'searching']92
8I can't eat all of this Satay BabatI can't eat all of this Saturday Bobbitt['i', \"can't\", 'eat', 'all', 'of', 'this', 'satay', 'babat']['i', \"can't\", 'eat', 'all', 'of', 'this', 'saturday', 'bobbitt']86
9Maxi Cash Cadbury and AjinomotoMexican Inn Camp Bowie and Angie nomoto['maxi', 'cash', 'cadbury', 'and', 'ajinomoto']['mexican', 'inn', 'camp', 'bowie', 'and', 'angie', 'nomoto']51
\n", + "
" + ], + "text/plain": [ + " truth \\\n", + "0 nine \n", + "1 De Fabio Factory and Apple \n", + "2 Oyakodon Yakizakana and Carrot Halwa \n", + "3 where can I find the best Red ruby \n", + "4 Kuih Kaswi waffle and Coconut Kuih \n", + "5 please tell me how to cook Satay Babat \n", + "6 six five two three nine seven nine three \n", + "7 Denis D Cotta Paine Eric and Chong Tze Chien \n", + "8 I can't eat all of this Satay Babat \n", + "9 Maxi Cash Cadbury and Ajinomoto \n", + "\n", + " output \\\n", + "0 nine \n", + "1 disable Factory in apple \n", + "2 oil Condon and carrot halwa \n", + "3 where can I find the best way to be \n", + "4 Quaker sweet waffle in Coconut Creek \n", + "5 please tell me how to cook saute Bobbitt \n", + "6 six five to three nine seven nine three \n", + "7 Dennis Dakota and Chong searching \n", + "8 I can't eat all of this Saturday Bobbitt \n", + "9 Mexican Inn Camp Bowie and Angie nomoto \n", + "\n", + " truth_split \\\n", + "0 ['nine'] \n", + "1 ['de', 'fabio', 'factory', 'and', 'apple'] \n", + "2 ['oyakodon', 'yakizakana', 'and', 'carrot', 'halwa'] \n", + "3 ['where', 'can', 'i', 'find', 'the', 'best', 'red', 'ruby'] \n", + "4 ['kuih', 'kaswi', 'waffle', 'and', 'coconut', 'kuih'] \n", + "5 ['please', 'tell', 'me', 'how', 'to', 'cook', 'satay', 'babat'] \n", + "6 ['six', 'five', 'two', 'three', 'nine', 'seven', 'nine', 'three'] \n", + "7 ['denis', 'd', 'cotta', 'paine', 'eric', 'and', 'chong', 'tze', 'chien'] \n", + "8 ['i', \"can't\", 'eat', 'all', 'of', 'this', 'satay', 'babat'] \n", + "9 ['maxi', 'cash', 'cadbury', 'and', 'ajinomoto'] \n", + "\n", + " output_split \\\n", + "0 ['nine'] \n", + "1 ['disable', 'factory', 'in', 'apple'] \n", + "2 ['oil', 'condon', 'and', 'carrot', 'halwa'] \n", + "3 ['where', 'can', 'i', 'find', 'the', 'best', 'way', 'to', 'be'] \n", + "4 ['quaker', 'sweet', 'waffle', 'in', 'coconut', 'creek'] \n", + "5 ['please', 'tell', 'me', 'how', 'to', 'cook', 'saute', 'bobbitt'] \n", + "6 ['six', 'five', 'to', 'three', 'nine', 'seven', 'nine', 'three'] \n", + "7 ['dennis', 'dakota', 'and', 'chong', 'searching'] \n", + "8 ['i', \"can't\", 'eat', 'all', 'of', 'this', 'saturday', 'bobbitt'] \n", + "9 ['mexican', 'inn', 'camp', 'bowie', 'and', 'angie', 'nomoto'] \n", + "\n", + " truth_count match_count \n", + "0 1 1 \n", + "1 5 2 \n", + "2 5 3 \n", + "3 8 6 \n", + "4 5 2 \n", + "5 8 6 \n", + "6 6 5 \n", + "7 9 2 \n", + "8 8 6 \n", + "9 5 1 " + ] + }, + "execution_count": 60, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Sample transcribed text vs truth\n", + "speaker = pd.read_csv(f\"output/2002.csv\")\n", + "speaker" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
sexracetruth_countmatch_count
0FCHINESE7760
1FCHINESE8464
2MCHINESE8053
3FINDIAN8053
4MCHINESE7045
...............
293MCHINESE7450
294MMALAY6852
295FCHINESE8258
296FCHINESE7952
297MCHINESE8257
\n", + "

298 rows × 4 columns

\n", + "
" + ], + "text/plain": [ + " sex race truth_count match_count\n", + "0 F CHINESE 77 60\n", + "1 F CHINESE 84 64\n", + "2 M CHINESE 80 53\n", + "3 F INDIAN 80 53\n", + "4 M CHINESE 70 45\n", + ".. .. ... ... ...\n", + "293 M CHINESE 74 50\n", + "294 M MALAY 68 52\n", + "295 F CHINESE 82 58\n", + "296 F CHINESE 79 52\n", + "297 M CHINESE 82 57\n", + "\n", + "[298 rows x 4 columns]" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# output to be passed into feat tests (subgroup disparity and min/max threshold)\n", + "df_speaker=pd.read_csv('../data/speech_sample_output.csv')\n", + "df_speaker" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "race\n", + "CHINESE 0.671469\n", + "INDIAN 0.690566\n", + "MALAY 0.683185\n", + "Name: rate, dtype: float64" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Sample accuracy stats across race\n", + "\n", + "tmp=df_speaker.copy()\n", + "tmp['rate']=tmp.match_count/tmp.truth_count\n", + "tmp.groupby('race').rate.mean()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "veritas", + "language": "python", + "name": "veritas" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/model_card_output/data/speech_to_text_example.proto b/examples/model_card_output/data/speech_to_text_example.proto new file mode 100644 index 0000000..3528fab Binary files /dev/null and b/examples/model_card_output/data/speech_to_text_example.proto differ diff --git a/examples/model_card_output/model_cards/speech_to_text_example.html b/examples/model_card_output/model_cards/speech_to_text_example.html new file mode 100644 index 0000000..0a29f35 --- /dev/null +++ b/examples/model_card_output/model_cards/speech_to_text_example.html @@ -0,0 +1,2097 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Model Card for Speech to Text Model + + + + + + + +

+ + Model Card for Speech to Text Model + +

+ +
+ +
+ +

Model Details

+ +

Overview

+ + Using an automated transcription tool, the government want to transcribe audio files recorded in trials for evidential purposes. The tool should be able to capture the various spoken accents and transcribe into text accurately so as not to bias against the defendant. The tool chosen here will be Google's Speech to Text Model evaluated on a set of audio files by National Speech Corpus. + +

Version

+ + + + + +
name: v1
+ + + + + + + + + + + + + + + + + + + +

Owners

+ + + + + +
  • Timothy, Product Owner(s)
  • + + + +
  • Swan, Model Developer(s)
  • + + + +
  • Jason, Reviewer(s)
  • + + + + + + + + + + + + + +

    Regulatory requirements

    + + MAS Fairness, Ethics, Accountability and Transparency (FEAT) principles + +
    + + + + + +
    + +

    Considerations

    + + + +

    Intended Users

    + + + + + + + + + + + + + +

    Use Cases

    + + + + + + + + + + + + + + + + + + + +

    Fairness Considerations

    + + + + + +
    + + + +
    + + + + + +
    + +

    Datasets

    + + + +
    + +
    + +

    National Speech Corpus Dataset

    + +

    Dataset taken from https://www.imda.gov.sg/programme-listing/digital-services-lab/national-speech-corpus

    + + + +

    Sensitive data

    + +
      + + + +
    • gender
    • + + + +
    • race
    • + + + +
    + + + +

    Sensitive data used in model

    + +
      + + + +
    • N.A. (Protected attributes are not trained in the speech to text model)
    • + + + +
    + + + + + +

    Justification

    + +

    N.A.

    + + + + + + + + + +
    + +
    + + + +
    + + + + + + + + + +
    + +

    Quantitative Analysis

    + + + + + + + +
    + +

    MSE - 712.513 (Test Set)

    + + + + + + + + + +
    + + + + + + + +
    + +

    MAE - 26.003 (Test Set)

    + + + + + + + + + +
    + +
    + + + +
    About 65% of the words are correctly transcribed
    + + + + + +
    + + + + + +
    + + None + +
    + + + +
    + + + +
    + +
    + + + + + + + +
    + + + + + + + +
    + +

    MSE across sensitive groups

    + + + + + + + + + + + + + +
    + + + +

    Min Max Threshold Test

    + + + + + +
    + + Description: + + Test if the mae of the subgroups within sex +is lower than the threshold of 27. + + +
    + + + + + +
    + + Threshold: + + 27 + +
    + + + + + +
    + + Result: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    maepassed
    sex_F26.143True
    sex_M25.891True
    + + + + + +
    + + + + + +
    Passed
    + + + + + + + +
    + +
    + + + + + +
    + + + + + +
    + + Mean Absolute Error across sex subgroups + +
    + + + +
    + + + +
    + +
    + + + + + +
    + + + + + + + +
    + + + +

    Min Max Threshold Test

    + + + + + +
    + + Description: + + Test if the mae of the subgroups within race +is lower than the threshold of 27. + + +
    + + + + + +
    + + Threshold: + + 27 + +
    + + + + + +
    + + Result: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    maepassed
    race_CHINESE27.141False
    race_INDIAN25.404True
    race_MALAY25.47True
    + + + + + +
    + + + + + +
    Failed
    + + + + + + + +
    + +
    + + + + + +
    + + + + + +
    + + Mean Absolute Error across race subgroups + +
    + + + +
    + + + +
    + +
    + + + + + +
    + + + + + + + +
    + + + + + + + + + +
    + + + + + + + + + + + +
    + +

    Fairness Analysis

    + + + + + + + +
    + +

    Equal mean squared error

    + + + +
    + + Segment: + + Gender and race + +
    + + + + + +
    + + Description: + + Disparity ratio of MSE of any 2 bins in the respective attribute should not be more than a factor of 1.1 + +
    + + + + + + + + + + + +
    + + + +

    Subgroup Disparity Test

    + + + + + +
    + + Description: + + Test if the maximum ratio of the mean squared error of any 2 +groups within race attribute exceeds 1.1. To +pass, this value cannot exceed the threshold. + +
    + + + + + +
    + + Threshold: + + 1.1 + +
    + + + + + +
    + + Result: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    race_mse_max_ratio
    01.136
    + + + + + +
    + + + + + +
    Failed
    + + + + + + + +
    + +
    + + + + + +
    + + + + + +
    + + Mean Squared Error across race subgroups + +
    + + + +
    + + + +
    + +
    + + + + + +
    + + + + + + + +
    + + + +

    Subgroup Disparity Test

    + + + + + +
    + + Description: + + Test if the maximum ratio of the mean squared error of any 2 +groups within sex attribute exceeds 1.1. To +pass, this value cannot exceed the threshold. + +
    + + + + + +
    + + Threshold: + + 1.1 + +
    + + + + + +
    + + Result: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    sex_mse_max_ratio
    01.023
    + + + + + +
    + + + + + +
    Passed
    + + + + + + + +
    + +
    + + + + + +
    + + + + + +
    + + Mean Squared Error across sex subgroups + +
    + + + +
    + + + +
    + +
    + + + + + +
    + + + + + + + +
    + + + + + + + +
    + + + + + + + + diff --git a/examples/sample-form-response-speech-to-text.json b/examples/sample-form-response-speech-to-text.json new file mode 100644 index 0000000..9ef78a9 --- /dev/null +++ b/examples/sample-form-response-speech-to-text.json @@ -0,0 +1,301 @@ +{ + "eventId": "4ceb5c3b-2878-4546-ad3e-9dab1147ba06", + "eventType": "FORM_RESPONSE", + "createdAt": "2021-09-26T07:14:49.410Z", + "data": { + "responseId": "100e0116-86e9-4625-8f80-dd539392104c", + "respondentId": "mZoZ90", + "formId": "mR4Nlw", + "formName": "Model Ops Card Builder", + "createdAt": "2021-09-26T07:14:49.000Z", + "fields": [ + { + "key": "question_3yXOqx", + "label": "Name", + "type": "INPUT_TEXT", + "value": "Speech to Text Model" + }, + { + "key": "question_waO1Wy", + "label": "Overview", + "type": "TEXTAREA", + "value": "Using an automated transcription tool, the government want to transcribe audio files recorded in trials for evidential purposes. The tool should be able to capture the various spoken accents and transcribe into text accurately so as not to bias against the defendant. The tool chosen here will be Google's Speech to Text Model evaluated on a set of audio files by National Speech Corpus." + }, + { + "key": "question_3X5jBY", + "label": "Intended Users", + "type": "INPUT_TEXT", + "value": "Judical System and Defendant" + }, + { + "key": "question_w8N9pP", + "label": "Intended Use Cases", + "type": "TEXTAREA", + "value": "Automate transcriptions and reduce human error and manpower in making court decisions" + }, + { + "key": "question_m6DYq5", + "label": "Version", + "type": "INPUT_TEXT", + "value": "v1" + }, + { + "key": "question_wbZMaE", + "label": "Product Owner(s)", + "type": "INPUT_TEXT", + "value": "Timothy" + }, + { + "key": "question_wArYkN", + "label": "Model Developer(s)", + "type": "INPUT_TEXT", + "value": "Swan" + }, + { + "key": "question_mBEYvY", + "label": "Reviewer(s)", + "type": "INPUT_TEXT", + "value": "Jason" + }, + { + "key": "question_wzEBxk", + "label": "Please select any regulatory guidelines which the model should be in compliance with", + "type": "CHECKBOXES", + "value": [ + "1f033a0c-1fa7-44a8-9309-0fe858283844", + "0a574b19-1a88-42e6-b4ba-b288b3fec5a4" + ], + "options": [ + { + "id": "0a574b19-1a88-42e6-b4ba-b288b3fec5a4", + "text": "MAS Fairness, Ethics, Accountability and Transparency (FEAT) principles" + }, + { + "id": "0e0e9512-6ea8-496e-9ee4-83dc1a23a5a7", + "text": "HKMA AI principles" + }, + { + "id": "5fcd3e36-042c-4c71-b76a-e9ec325f6205", + "text": "Bank of England and FCA's guideline" + } + ] + }, + { + "key": "question_wzEBxk_0a574b19-1a88-42e6-b4ba-b288b3fec5a4", + "label": "Please select any regulatory guidelines which the model should be in compliance with (MAS Fairness, Ethics, Accountability and Transparency (FEAT) principles)", + "type": "CHECKBOXES", + "value": true + }, + { + "key": "question_wzEBxk_0e0e9512-6ea8-496e-9ee4-83dc1a23a5a7", + "label": "Please select any regulatory guidelines which the model should be in compliance with (HKMA AI principles)", + "type": "CHECKBOXES", + "value": false + }, + { + "key": "question_wzEBxk_5fcd3e36-042c-4c71-b76a-e9ec325f6205", + "label": "Please select any regulatory guidelines which the model should be in compliance with (Bank of England and FCA's guideline)", + "type": "CHECKBOXES", + "value": false + }, + { + "key": "question_wzEBxk_1f033a0c-1fa7-44a8-9309-0fe858283844", + "label": "Please select any regulatory guidelines which the model should be in compliance with (Not applicable)", + "type": "CHECKBOXES", + "value": true + }, + { + "key": "question_n0V4jj", + "label": "Name of dataset", + "type": "INPUT_TEXT", + "value": "National Speech Corpus Dataset" + }, + { + "key": "question_wM10jE", + "label": "Description of dataset", + "type": "TEXTAREA", + "value": "Dataset taken from https://www.imda.gov.sg/programme-listing/digital-services-lab/national-speech-corpus" + }, + { + "key": "question_wzEVQ0", + "label": "Protected attributes in dataset", + "type": "TEXTAREA", + "value": "gender, race" + }, + { + "key": "question_w5XNqE", + "label": "Protected attributes in production", + "type": "TEXTAREA", + "value": "N.A. (Protected attributes are not trained in the speech to text model)" + }, + { + "key": "question_3Eqpo2", + "label": "Justification of use of protected attributes", + "type": "TEXTAREA", + "value": "N.A." + }, + { + "key": "question_wdb72K", + "label": "Name of dataset 2", + "type": "INPUT_TEXT", + "value": null + }, + { + "key": "question_mJ1yEz", + "label": "Description of dataset 2", + "type": "TEXTAREA", + "value": null + }, + { + "key": "question_mYj2zJ", + "label": "Protected attributes in dataset 2", + "type": "TEXTAREA", + "value": null + }, + { + "key": "question_mDqLvZ", + "label": "Protected attributes in production (dataset 2)", + "type": "TEXTAREA", + "value": null + }, + { + "key": "question_3jeK8Y", + "label": "Justification of use of protected attributes (dataset 2)", + "type": "TEXTAREA", + "value": null + }, + { + "key": "question_3laplv", + "label": "Name of dataset 3", + "type": "INPUT_TEXT", + "value": null + }, + { + "key": "question_wgbB8M", + "label": "Description of dataset 3", + "type": "TEXTAREA", + "value": null + }, + { + "key": "question_mRWOBj", + "label": "Protected attributes in dataset 3", + "type": "TEXTAREA", + "value": null + }, + { + "key": "question_wo9JVx", + "label": "Protected attributes in production (dataset 3)", + "type": "TEXTAREA", + "value": null + }, + { + "key": "question_3yXO86", + "label": "Justification of use of protected attributes (dataset 3)", + "type": "TEXTAREA", + "value": null + }, + { + "key": "question_3je4aQ", + "label": "What is the key metric used in evaluating the model's performance?", + "type": "INPUT_TEXT", + "value": "Mean Squared Error" + }, + { + "key": "question_mOQRBg", + "label": "2nd metric (if applicable)", + "type": "INPUT_TEXT", + "value": "Mean Absolute Error" + }, + { + "key": "question_mVpr1y", + "label": "3rd metric (if applicable)", + "type": "INPUT_TEXT", + "value": null + }, + { + "key": "question_mKp4dz", + "label": "Who are the individuals and groups that are considered to be at-risk of being systematically disadvantaged by the system?", + "type": "TEXTAREA", + "value": "gender and race" + }, + { + "key": "question_31WYq4", + "label": "Expected Benefits", + "type": "TEXTAREA", + "value": "Reduce manpower and human error on transcription services. A more accurate model than current system will aid the judicial system in making better court decisions" + }, + { + "key": "question_nGeMBk", + "label": "Expected Harms", + "type": "TEXTAREA", + "value": "Incorrectly transcribed/interpreted texts can impact on the defendant/ruling negatively." + }, + { + "key": "question_mJ1r5z", + "label": "Mitigation strategies", + "type": "TEXTAREA", + "value": "Improve upon the existing model by further training and enhancements" + }, + { + "key": "question_nPRoBe", + "label": "Type of fairness analysis conducted", + "type": "INPUT_TEXT", + "value": "Equal mean squared error" + }, + { + "key": "question_3Eqpbr", + "label": "Segment of analysis", + "type": "INPUT_TEXT", + "value": "Gender and race" + }, + { + "key": "question_nrDXrR", + "label": "Description of fairness analysis", + "type": "TEXTAREA", + "value": "Disparity ratio of MSE of any 2 bins in the respective attribute should not be more than a factor of 1.1" + }, + { + "key": "question_w4aZNY", + "label": "Type of fairness analysis conducted 2", + "type": "INPUT_TEXT", + "value": null + }, + { + "key": "question_3jeK94", + "label": "Segment of analysis 2", + "type": "INPUT_TEXT", + "value": null + }, + { + "key": "question_w2j2x9", + "label": "Description of fairness analysis 2", + "type": "TEXTAREA", + "value": null + }, + { + "key": "question_3xVGN5", + "label": "Segment of analysis 3", + "type": "INPUT_TEXT", + "value": null + }, + { + "key": "question_mZ9xRB", + "label": "Type of fairness analysis conducted 3", + "type": "INPUT_TEXT", + "value": null + }, + { + "key": "question_3NqzBj", + "label": "Description of fairness analysis 3", + "type": "TEXTAREA", + "value": null + }, + { + "key": "question_n0V0Q9", + "label": "Please enter your email", + "type": "INPUT_EMAIL", + "value": "timothy.lin@cylynx.io" + } + ] + } +} \ No newline at end of file diff --git a/examples/speech_to_text_example.ipynb b/examples/speech_to_text_example.ipynb new file mode 100644 index 0000000..dbcc0d5 --- /dev/null +++ b/examples/speech_to_text_example.ipynb @@ -0,0 +1,2605 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import json\n", + "from IPython import display\n", + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.metrics import mean_squared_error, mean_absolute_error\n", + "\n", + "# parent directory to work with dev\n", + "sys.path.insert(0, '..')\n", + "import verifyml.model_card_toolkit as mctlib\n", + "from verifyml.model_card_toolkit import model_card_pb2, ModelCard\n", + "from verifyml.model_card_toolkit.utils.tally_form import tally_form_to_mc\n", + "from verifyml.model_tests.utils import plot_to_str\n", + "from verifyml.model_tests.FEAT import (\n", + " SubgroupDisparity,\n", + " MinMaxMetricThreshold,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Read output with truth and prediction columns" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# Read output\n", + "# truth column -> total no. of words spoken by the individual\n", + "# prediction column -> total no. of words correctly transcribed by google's speech to text model model\n", + "\n", + "output = pd.read_csv(\"../data/speech_sample_output.csv\")\n", + "output.rename(columns={'truth_count':'truth', 'match_count':'prediction'},inplace=True)\n", + "mse_test = round(mean_squared_error(output.truth, output.prediction),3)\n", + "mae_test = round(mean_absolute_error(output.truth, output.prediction), 3)\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
    \n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    sexracetruthprediction
    0FCHINESE7760
    1FCHINESE8464
    2MCHINESE8053
    3FINDIAN8053
    4MCHINESE7045
    ...............
    293MCHINESE7450
    294MMALAY6852
    295FCHINESE8258
    296FCHINESE7952
    297MCHINESE8257
    \n", + "

    298 rows × 4 columns

    \n", + "
    " + ], + "text/plain": [ + " sex race truth prediction\n", + "0 F CHINESE 77 60\n", + "1 F CHINESE 84 64\n", + "2 M CHINESE 80 53\n", + "3 F INDIAN 80 53\n", + "4 M CHINESE 70 45\n", + ".. .. ... ... ...\n", + "293 M CHINESE 74 50\n", + "294 M MALAY 68 52\n", + "295 F CHINESE 82 58\n", + "296 F CHINESE 79 52\n", + "297 M CHINESE 82 57\n", + "\n", + "[298 rows x 4 columns]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Word count plot" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAA4VUlEQVR4nO29ebgcVbX3//nmEJKTBMjAgUtCZmNyEa4M5woYrhKiRpyIeFERfMEB9DrgGE00jyNKXvE6XLmKXLiCIpFBCPjCD0QGQZRAwhQEIhIyEJCEkDAlQEjW74+qPumc1K6u6lQPp3t9nqef7tpVtffau7prde2111oyMxzHcRwHoF+jBXAcx3GaB1cKjuM4Tg+uFBzHcZweXCk4juM4PbhScBzHcXpwpeA4juP04EqhDEnvlrRK0vOSDqpTmzdL+mg92qo1ksZJMkm7NFqWeiPpdElPSfpHA2Vo+fGX9P9JOin+fLKkPxVUb0uMXdyHV+1MHU2nFCQtl/SkpMFlZR+VdHMdmv8+8CkzG2Jmd9ehPacgyn7UV/cqv1DSN2rc9mjgC8B+ZvZPtWyrFkj6hqQLGy1HFszsaDO7oNFyNAO1+kPZdEohZhfgMw1odyzw1wa02+do4n9Uh0maWuc2xwLrzGxNvRqs5/groqH3imaQYWdJumbN+Dtq1kE+E/iipKFJOyW9XtKdkp6J31+fpVJJ/STNlbRC0hpJv5S0h6QBkp4HOoB7JT2ScK4k/TA+7xlJ90naP953vqSzJV0v6TlJf5Q0Nq+8kvaJ6/1ivH2YpD9L2iDpXklHlh17s6RvS7otbvP3kvYM1PtHSe+JPx8R/6N+W7z9Jkn3pI1PvK/0T/wjklYCN0rqkPT9eNpkGfD2Xu2eLGlZLN+jkk5IkG2kpE2ShpeVHRTX2V/Sq2L5n4nLLk7qYxnfA04P7ZR0iqS/S3pa0lWSRlaor3TeHvF4rI3HZ248Xm8CrgdGKpp2PD/h3GYe/7cCXwHeF8t/b1x+s6TvSLoN2AhMkPQhSQ/G9S2T9LGyeo6U9JikL8SyPyHpQ2X73ybpgfjc1aXveLzvGEn3SHpW0iOxTCEZev87lqSfxN+PhyRN73XNzotlWa1oiq8j3pc6dgnjNFrS5fH1XyfprCqv2cmKfrM/lPQ08A1F95/vS1qpaJbkbEmdaeMj6TvAvwFnxdftrF7y/mtc1y5lZe8pfddSMbOmegHLgTcBlwOnx2UfBW6OPw8H1gMfJHqiOD7eHpGh7g8DfwcmAEPiNn5Vtt+AVwXOnQEsBoYCAv4Z2Cfedz7wHPAGYADwY+BPWeQFbo77Nw74G3BqXD4KWAe8jUh5vzne7io77xHg1UBnvD0vIPu3gJ/En78Sn/d/y/b9uNL4xPIZ8EtgcNzmx4GHgNFxP2+Kj9klPuZZYHJ8/j7AawLy3QicUrZ9JnB2/Hk+8NV4DAYCRwTqKMk3BFgNvCkuvxD4Rvz5KOAp4OD4Ov0EuCXj9/KXwJXAbmXX6iPxviOBx1LObfbx/wZwYa+ym4GVwGvi+voT3TgnEn3/30h0oz64bAxeifvTn+h7uxEYFu9/Avi3+POwsvNeBzxD9P3uR/S9n5Iiw83AR+P9J8dtfi7e9764ruHx/gXAz+Ox2Au4A/hYvC84dgnj0wHcC/wwrqvne1jFNSvJ/Om4T53Aj4CrYjl2A34HnJFxfD7aS9aeexjwAHB02b4rgC9U/K7X6uZe7YttSmH/eDC62F4pfBC4o9c5fwFOzlD3DcAnyrYnA5tLXwTSlcJRRDeCw4B+vfadD/ymbHsIsCX+wqXKG1/YH8T9Pr7smC9TprDisuuAk8rOm1u27xPAtQHZpwP3xZ+vjcfz9nj7j8Cxlcan7As+oWz/jcDHy7bfwvY3pQ3Ae4DOCtflo8CN8WcBq4A3xNu/BM4B9q1Qx7iytj9R1r9ypXAe8L1e12kzMK5C3R3AS0Q2g1LZx8q+k0eSrhSaffy/QbJS+FaF8xYAnykbg02U3VSBNcBh8eeV8Zjt3quOnwM/DNS/gwzsqBQeB1S2/w6i39ze8TXrLNt3PHBTpbFLkONwYG1gX95rdjKwsmxbwAvAxF7tPZpxfNKUwpeBX8efhxMp6X3SrqmZNe30EWZ2P/D/gNm9do0EVvQqW0GkQSvR+9wVRBdv7wzy3AicBfw38KSkcyTtXnbIqrJjnweejtvLIu8JRP9uLysrGwscp2jqaIOkDcARRP/4SpSvdNlIdJNL4i/AqyXtDRxIdKMdrWi66XXALfFxWcZnVdnnkb22e841sxeI/rl9HHhC0tWSpgTkuww4XNFUzhuIvti3xvu+RPTDuUPSXyV9OFBHOf8D7C3pnb3Kt+tffJ3WUfm7syewKzuOTZbvHDT/+IcorxtJR0u6XdHU2waip4HyKct1ZvZK2Xb5d/I98fErFE2nHR6XjyZ6csokQwKrLb7rxawgGpexRE8PT5T9fn5O9MQAKWOXwGhgRa++lch7zXpvdwGDgMVlcl4bl5faThufNC4E3ilpCPBe4FYze6LSSU2rFGK+DpzC9j++x4kueDljiG6qleh97hiiR7knswhjZv9lZocQPc6+GphVtnt06UN8EYbH7WWR9xtE0xoXleY8ib44vzKzoWWvwWY2L4usveTeSDT19RngfjN7Gfgz8HngETN7Kj40y/iU/wCfoKzf8fHl7V5nZm8mUmQPEd2sk+TbAPye6Iv7AWB+6YduZv8ws1PMbCTRP82fqsKSOzPbDHwT+DaRQimxXf8UrXAbQeXvzlNE//56j02W71zTj3+vOhPLJQ0Afku0Qm9vMxsKXMP24xvEzO40s2OIbsoLgEviXauIpqSCp1aoepSkchnGEI3jKqInhT3Lfj+7m9lr4uNSx64Xq4AxSjYK571mvbefInrCek2ZnHuYWUmZpo1P6tiY2WqiPyTvJnp6+lXa8SWaWimY2d+Bi4HTyoqvIfrX9QFJu0h6H7Af0VNFJeYDn5M0Pr5xfxe4OPAPYDtiw82hkvoTPe69SDRFVOJtioyIuxLdjBaa2aqM8m4GjiN65P+VolUWJS0/IzaKDVRkzNs3Qz+T+CPwqfgdokfP8m3IPz6XAKdJ2lfSMMqe6iTtLeld8Y33JeB5th+v3lwE/B+if5QXldVzXFmf1xP9ENLqKfErIrvBW3u18SFJB8Y3ue8SXaflaRWZ2Za4r9+RtJuiRQSfJ7pGWWnm8X8SGKf01T27Eo3nWuAVSUcTTblURNKukk6QtEessJ8tk+U8omsyPTbajsr5RLMX0Rj0l3Qcka3vmvgf8e+B/5S0e1z3RElvjM8Ljl0CdxApkXmSBse/xdIKt6rvKQBmtpVIWf9Q0l4A8RjMiA9JG58niWwZafyS6Gn7ACKbQiahmupFbFMo2x5NdAO+uazsCKJ/Xs/E70eU7fsrcEKg7n7A14i071qiH/WwpPm4hHOnA/cR/bieAn4NDIn3nQ+cTbQK5Xmi6YDxGeW9mW1zpAOBP8T19QMOJbppPB3LezUwpvd5tm2u8k8p4zoj7t8b4+394+33ZRkfyubsy47fhcj4tg54FPhk6Riif6d/jPu8IZZ3vxT5OomM9X/tVf49on/kzxM9Rp8aOD9JvvfGZd8oK/t4XM/TRIp537h8TNzGmED9w+LxWBuPz9eIbUtUsCk0+/gTPS39iUjp3pX0/YrLPkl0I9pApHR/w7bFIDuMAdvsg7sSTYmsJ1IId7L9b+DdRL+t54iMtjNSZOgpI/rO30Y0rfsMkc3vLWXH7gH8DHgs3n838P5KYxcYozFETzjriH7//1XlNTuZXr9Tot/9d4Fl8fg8CJyWYXwOj/u8vkye7e5hRFNTzwIXVLr3ll6KT3R2AkXLEB8zs7mNlsVxHKccRUvsP2Zmf8hyfFNPHzmO4zjVo8g/xohWW2Wi6bzpHMdxnJ1HUWig/YAPWmS7yHZeLaePJH2OaE22AUuADxHNcV1MNN+2HHivma2vmRCO4zhOZmo2fSRpFNGqoW4z25/IAej9RFb+G8xsEpHjR5rV33Ecx6kjtZ4+2gXolLSZ6AnhcWAO0UoFgAuIVhN8Oa2SPffc08aNG1czIR3HcVqRxYsXP2VmXZWP3EbNlIKZrZb0fSL39k3A783s95L2ttirzsyeKK3N7Y2kU4FTAcaMGcOiRYtqJarjOE5LIinNUzuRWk4fDQOOAcYTuYIPlnRi1vPN7Bwz6zaz7q6uXIrOcRzHqZJaLkl9E1FQp7UWeTFeDryeKG7QPhCFiiYKmuU4juM0AbVUCiuJEp4MimOTTCfy1LsKOCk+5iSicMSO4zhOE1BLm8JCSZcBdxEFiLqbKATyEOASSR8hUhzH1UoGx3EcJx81XX1kZl8ninRazktETw2O4zhOk+EezY7jNJQFd6/mzOuW8viGTYwc2smsGZOZeVDWVBVO0bhScBynYSy4ezVzLl/Cps1RJO3VGzYx5/IlAK4YGoQHxHMcp2Gced3SHoVQYtPmLZx53dIGSeS4UnAcp2E8vmFTrnKn9rhScBynYYwc2pmr3Kk9rhQcx2kYs2ZMprN/x3Zlnf07mDVjcoMkctzQ7DhOwygZk331UfPgSsFxnIYy86BRrgSaCJ8+chzHcXpwpeA4juP04ErBcRzH6cGVguM4jtODKwXHcRynB1cKjuM4Tg+uFBzHcZweXCk4juM4PbhScBzHcXpwpeA4juP04ErBcRzH6cGVguM4jtODKwXHcRynB1cKjuM4Tg+uFBzHcZweKuZTkPQp4Ndmtr4O8jiO0wdYcPdqT4zTomR5Uvgn4E5Jl0h6qyTVWijHcZqXBXevZs7lS1i9YRMGrN6wiTmXL2HB3asbLZpTABWVgpnNBSYB5wEnAw9L+q6kiTWWzXGcJuTM65ayafOW7co2bd7CmdctbZBETpFksimYmQH/iF+vAMOAyyR9r4ayOY7ThDy+YVOucqdvkcWmcBpwEvAUcC4wy8w2S+oHPAx8qbYiOo7Tm0bO6Y8c2snqBAUwcmhnXdp3akuWJ4U9gWPNbIaZXWpmmwHMbCvwjppK5zjODjR6Tn/WjMl09u/YrqyzfwezZkyuS/tObQkqBUnDJQ0HfgQ8V9ouK8fMHqyTnI7jxDR6Tn/mQaM449gDGDW0EwGjhnZyxrEH+OqjFiFt+mgxYEDSaiMDJtREIsdxUmmGOf2ZB41yJdCiBJWCmY2vpyCO42TD5/SdWlLRpiDphixljuPUB5/Td2pJ8ElB0kBgMLCnpGFsm0baHRhZB9kcx0mgNG3jHsVOLUizKXwM+CyRAljMNqXwLPDftRXLcZw0fE7fqRVpNoUfAz+W9Gkz+0kdZXIcx3EaRBY/ha2ShpY2JA2T9IlKJ0maLOmestezkj4bL2m9XtLD8fuwnemA47QjC+5ezdR5NzJ+9tVMnXejxx1yCiOLUjjFzDaUNuJoqadUOsnMlprZgWZ2IHAIsBG4ApgN3GBmk4Ab4m3HcTLSaOc1p7XJohT6lUdGldQB7JqznenAI2a2AjgGuCAuvwCYmbMux2lrGu285rQ2WZTCdcAlkqZLOgqYD1ybs533x+cB7G1mTwDE73slnSDpVEmLJC1au3ZtzuYcp3VpBuc1p3XJohS+DNwI/AfwSaIpn8xB8CTtCrwLuDSPYGZ2jpl1m1l3V1dXnlMdp6UJOam585pTBBWjpJrZVknnAzeaWTXPp0cDd5nZk/H2k5L2MbMnJO0DrKmiTsdpC5Kioc6aMZlZl97L5q3Wc1z/fnLnNacQsng0vwu4h3jKSNKBkq7K0cbxbJs6AriKKBQ38fuVOepynLYhZFBetOLpHSOSeT5EpyCyTB99HXgdsAHAzO4BxmWpXNIg4M3A5WXF84A3S3o43jcvs7SO00aEDMrzF65i8xbbrnzzFnNDs1MIFaePgFfM7JlqUjOb2UZgRK+ydUSrkRzHSSFkON5illjuhmanCLIohfslfQDokDQJOA34c23Fcpz2IZRFLRQNtUNKVAzVGpobmcXNaT6yTB99GngN8BJwEfAMUUwkx3F2kjRHtFA01OMPHV1YlFR3hHN6k/qkEDuqXWVmbwK+Wh+RHKd9SHNEu232UT3H9P4X3z12eCH/7tPa96eF9iRVKZjZFkkbJe1hZs/USyjHaRcqOaKFoqEWFSXVHeGc3mSxKbwILJF0PfBCqdDMTquZVI7TJjQ6i1qj23eajyxK4er45ThOBvIYbmfNmMycy5dsN4VTbh+otRF41ozJzLrs3u2WuPbvKN4Rrl2M2a3Qzyw2hQ/GNgXHcSpQMtyWbvIlwy0QnAaCZLtB3rqqpvdCpuQVr1VTt340mFbpZ+rqIzPbAmyUtEed5HGcPk01EUxnHjSK22YfxaPz3s5ts4/aTlHUOhrqmdct3S5cBsDmrcU6wrVLVNdW6afbFBynQIo03NbDCNwqbTQDrdJPtyk4Tgp554jTDLdF1lUPeYuiXYzZrdLPis5rZnYBUUC7xfHrorjMcVqaahy7pk1JDvM+bkRn7rpCzmshI3A18uZtoxrq0UYz0Cr9zBIl9UjgYeC/gZ8Cf5P0htqK5TiNp5o54pseSk4Idfuy9VXZGs449gBGDe1EwKihnZxx7AHBf/7V2jPytFEN9WijGWiVfmaZPvpP4C2lXAqSXk305HBILQVznEZTzRxx0UHs8jipVTunXZQjXKPbaAZaoZ9ZYh/1L0+uY2Z/A/rXTiTHaQ6qyXAW2tcRiDJc9Nx9rdtwWp8sSmGRpPMkHRm//ofItuA4LcOCu1czdd6NjJ99NVPn3ZgakC5tjrgeQezS5O3fb3vl4xnZIpLGy0kmi1L4D+CvRCGzPwM8AHy8lkI5Tj0JGWiB3HPEoXnl02ceUNh8s2dky4dHgs2HLDDX2XOANBh4MXZkK3k5D4gT6NSF7u5uW7RoUb2ac9qMqfNuTFxKOGpoZ0+k0mYiJG8oz0Kz9qNe9LXrWySSFptZd55zsjwp3ACUT0p2An/I04jjNDN9zenIM7Llo69d30aTZfXRQDN7vrRhZs/HuZcdpyWoxuGsmsBnRZ1Tr4xsRdLIQHGt4lRWL7I8Kbwg6eDShqRDAFexTsuQ1+Fs7oIlueeoq5nXDp0zbUpXXYzZRdHoOf1WcSqrF1mUwmeBSyXdKulW4GLgUzWVynHqSF6Hs/kLV+V2EqvGsSx0zk0Pra25MbtIGh0orlWcyupFxekjM7tT0hRgMtFahofMbHPNJXOcOpF3jr6aufsiHeEe37Cp5hnZiqQZ5vSbcVyalSxPCpjZZjO738yWuEJwWo28DmfVOKIV6QjX1+bCW6Uf7UImpeA4zUhRDkkhp6/QHH2lufugY1lHrzYqZDhLO6eavjfKgavSnL47ljUXWVYfOU7TUXiWqwSnr+6xw+keOzxx1UyoPCTXew4ZVV2Gs4RzFq14mt8uXp2r743MCtYU2eWczASd18pXHCVhZnfVRKIE3HnN6U2RDkn1qKsax7J61NVoB65mlatVqMZ5Le1J4T/j94FAN3Av0f+pfwEWAkdUI6TjFEGzZjgr0rGsHnU12oGrWeVqZ4JKwcymAUj6DXCqmS2Jt/cHvlgf8ZxWoygnpiIdkupRV5pjWWhMinRSK9qBqx7XsZEOb+1MFkPzlJJCADCz+4EDayaR07IU6cRUpENSPeqa0JUcBGDQrv2CYzJuRPINe0LXoMKit1bTx3pcx2lTujyIXYPIohQelHRuHDb7jXHo7AdrLZjTehTpxFSkQ1I96lq2Njl+5MNrXgiOye3L1iees2ztxsKit1bTx3pcx5seWttQh7d2JkuU1IFE4bNLKThvAX5mZi/WWLYe3NDcGoyffXXighsBj857e73FqSvjZl+d63iRvjhpeQPHqx7XsZ2/K0VStKEZADN7UdLZwDXlGdgcJy/tHJgsZAcIMXJoJ/945sXEc0LOc/WiHtexnb8rjaaiUpD0LuBMYFdgvKQDgW+Z2btqLJvTYsyaMZlZl97L5q3bbnT1zAxWZGTTuQuWMH/hKraY0aHI0e30mQcAJO47/tDRXHj7yh3qnzpxOHc8uj5xTBateDrxnOMPHV2oETZvJNhZMyZv51sAxQeYq0cbzU6jDO1ZnNe+DrwOuBnAzO6RNK6GMjmtTIMyg1XjJBU659JFK7ntkad7jttitt3Nu/xzad+Jh43hxMPG7KAsuscO547lvWwH8ZiUlEzSOUU5fIX6mMVBrpY3rHq00cw00qkvi01hoZkdKuluMzsoLrvPzP6lppKV4TaF1qCRjkrVtB06J0RpWic05fPIGW+rqVyNdrZziqOoa1wTmwJwv6QPAB2SJhHlav5znkYcBxrrqFRklNIQaTaDvA5n9XJE8yxuzUkjfytZlMKnga8CLwHzgeuAb2epXNJQ4Fxgf6LFFB8GlhLlZBgHLAfea2bJa++clqLoDGehef2k8moMl6FzQlR6UqhGrjyZ1+rlbOfUnkYa2iv6KZjZRjP7KjAdmGZmX82xHPXHwLVmNgV4LZF/w2zgBjObRJT/eXZ1ojt9jbyOSmkZzuYuWMKFt6/suXGV5u7f/IObE8sH7Zr8VQ9lXUuTd+/ddk08fkLXII4/dHRwX5Jcr2zZknj8uBGduTOvVWOEDfV/zyH9g3I5taeR2eIqKgVJ/yppCXAfsETSvXFKzkrn7U7k23AegJm9bGYbgGOAC+LDLgBmVie609fI66iUluFs/sJViW08vOaFXOWhrGtp8j71fHJKkWVrN3L6zAM48bAxPU8NHRInHjYm6Lz25HMvJ5bfvmx97sxr1RggQ/1Pk8upPY3MFpfF0Hwf8EkzuzXePgL4aSVDc7x09RzgAaKnhMXAZ4DVZja07Lj1ZjYs4fxTgVMBxowZc8iKFSuy98rpU4QclUJUcuzKQzXOUGmOaCGnsrzOaxDuZz2cxNJopOOck49qDM1Zwlw8V1IIAGb2J+C5DOftAhxM5P18EPACOaaKzOwcM+s2s+6urvAjvtP3yZv5bOTQzsIcuKqZo82bka3SvtDx9chYlreuRjvOObUnqBQkHRznVLhD0s/LYh/9lNhnoQKPAY+Z2cJ4+zIiJfGkpH3iNvYB1uxUD5w+RTArWSDzWe8vaD+i+dbQ3P2kvQYnlk+dODxYV0iuUHmo7VB52r6QvMcfOjp1Xjkk79wFS5g45xrGzb6aiXOuYe6CJUnV9xBqY+rE4bn64bQOWfIplPh62eeKT5xm9g9JqyRNjsNjTCeaSnoAOAmYF79fmU9kp6+SmpUswant0bXPs7VX8VaizGOPrn0+2E6/+LjybXqVldcF5HLgOuPYZKeykrNZEt1jh3PR7St3kOvQCSNY9tRGtpR5NHf0E91jhwcduELypjnVhWRLcxJL89p2WpeKNoWdqjyyK5xLFCJjGfAhot/CJcAYYCVwnJk9HaoD3HmtVcjrKBUi7/GV6vqnPQbW3IGrHlnUQoQc55zWp1DnNUknmtmFkj6ftN/MflCpcjO7hyhrW2+mZ5bQaRnyOkqFKEohlOqqhwNXPbKohShyvJzWJ236qDTZuVs9BHFan7yOUiEa/aRQyTibx0mtyCxqIdrJOOzZ2naeoKHZzH4uqQN41sy+2ftVRxmdFiFk1Dxswg4rkgHYfUBHYvmErkFBQ+jAjnw3wMG79gs6ZIUcuNIc3kJOdSEntWqyqIXaTzNatwNFZoRrZ1KXpJrZFsBDZDuFEHLIWb4u+V/vsy8l30iXrd3Ir085fAfFMHXicF7cku8J4tmXtgQdskIOXGkObyGnulBd1WRRC7W/8eWtiY5z7WIcLjIjXDuTJfbRnyWdRRSvqMct1MzuqplUTssy86BRO9zwPnfxPbnqKP0L//Uph++wrxonsbxTUWlz+tXYR5LGpJr2H9+widNnHtA2SqA3jQwi10pkUQqvj9+/VVZmgMfPdQqh0XPk1WRFK6quavriWcmS8XEphizpOKfVQxCnsRSZlSzvObNmTObzl9xD2VJ9+gkOnzB8u3X3JUpz5El1TZ2YfM7ADiVOLU3aazCHThiROytaqO+hDGuT9hqcGH+pmvn+ojPYVROhthnxbG3FkCUd53eB78XB7JA0DPiCmc2tsWxOnSgyK1k15xw8Zo/tFALAVoPxXUMY3zUk0YEqVNe+wwYmtr3HoP68mDCvf+iEEXSPHc78O1bt4EA2vmtIYla0tKxkadnSkhRf99hkg3lFCspgtzOZ15qNds/WVhRZAuL1ZFwrK7vLzA6uqWRluPNabalX9q8ina7y1pXWRt4lqUU6nBXpCNdopzqn+ahVQLwOSQPKGukEBqQc7/Qx6pX9q0inq6KMh9U4rxXpcFakI1yjneqc1iCLoflC4AZJv2Bb9rQL0k9xqqFRc7hFZiUbObQzGDOnGoNyUXWltVHUk0K12dLyXvdmyLzW1+wNTnayZF77HnA68M/Aa4Bvx2VOgTTS8SbkvJWWZSvkQDVo136JzltzFyzJnclscEpdofZDzmu7BObcD5swLJiVbXCgPC0rWd5saaGsc2nXPdT3NKe6EKFrcvyho1MjtLqTWOuSJfPaYOD3ZvZFoqQ5AyQl/yqcqmmk403IeSsty1bIgSqU4Wz+wlW5M5mFnNfmL1wVbD/kvPZKYCZq+bpNQZlD7ReZLS2UdS7tuof6nuZUFyJ0TU6feUDQqc6dxFqbLNNHtwD/Fq86+gOwCHgfcEItBWs3Gul4k3fuHKq3DyQ5an22Cue1osalyPFNk+vxDZtyOe7Vyz4BydckrdydxFqbLIZmmdlG4FjgJ2b2bmC/2orVftQjy1aIajKJFZmxq8isZHkpcnyryZZWzXVv5HelGdp3aksmpSDpcKIng1IMgSxPGE4O0rJs1ZpKmcSC2dJ6zd/371BVGbuqzUqWlK0t1H6ovOTwFmo/b1ay0LiEsqVVc90b+V1phvYbTSjrXauQRSl8BpgDXGFmf5U0AbiptmK1H6G53Xqs6Ogem5yqsnvs8KBRcdGKp3fMv2dwXPeY3EHZuscOp6PXDb6jnzh0wgh6FW/v8JXgwDW+a0jiOYt7O6HFfG3BkmBwves/f2TiNRnfNSTYFyBxXBateDpxHIHc172R35VmaL+RtIORvaaZ14rCnddqS5ozFNCUWcnyypXG8nlvz3X8xDnXBJeq1iOLm9M4inQcrAeFZl4rq/TVwBeBceXHm1nzjYBTFdUYDps1K1k9soylGebdGay1aQcjexbbwKXA2US5lpPX6Dl9hryOVZDvH3klY2Oe9iu1UdSTQl5505zais7i5k5izUU7RGLNYlN4xcx+ZmZ3mNni0qvmkjmFk9exqhTBNK9zU972Q05yIeexQbv2Czqc7Tmkf6JcIee1tExtIXlDmeKOP3R00IHssAnDChuvVpq/7msU6TjYrGRRCr+T9AlJ+0gaXnrVXDKncPI6VpXWqed1bsrbfshJLuQ89vCaF4IOZ08+93KiXHvvkax4RuyWHFU1Td7l6zYFjekhB7Ll6zYVNl7uJNY4inQcbFayTB+dFL/PKiszYELx4ji1JK9jVYm8zk152y/aDlAPJ7FQhrNqxzhv+05jaIdrkiX20fiElyuEPkijnY5C7RSdSS1P20U7iRU5xo2+Xs6OtMM1yeSEJml/Ii/mnmdtM/tlrYRyakPRmanyZuwKZQx73+tGc9HClTskoJnYlZytrORTkJRhberE4YmRVStlKwudExqvvH2sZDsoqi6ntrRDdrcsS1K/DhxJpBSuAY4G/gS4UuhjFJmZqpqMXUCiw9nCZesSM6/ttdsA9tptwHY3/6kTh/PrUw4H4IT/+csO+8Z3DdkuHWYpsuqja58PZiubu2BJ4jkQOZb1Hi8gmHUu1McQaRns8tbl1J52yO6WJfPaEuC1wN1m9lpJewPnmtk76yEguPNaM1Kkw1kaRTmWhRg1tJN/PPNicIlpUua3apz9Qs5NRdblOL2pifMasMnMtkp6RdLuwBrcyNz2NKuTVl6j9ePxcs88ddUjU12RGewcJw9ZlMIiSUOB/wEWA88Dd9RSKGd76uHAlNZGrR3O0ghlXgvty+u8NrLCk0JRzn5pBsqi6nKcIkhVCpIEnGFmG4CzJV0L7G5m99VDOCd9zrkoxVBpXjtp33sOGbWd7QAig9u+wwYmGodLDmpJN7mBHUpMjrP7gI7gXD+QuG/3AR2J/g2h8mlTunh07fOJRusJXYMS+37wmD0S+1GqK2lfyEFv2pSu7fpRXt49dnjLGzWd5iOLTWGxmR1SJ3kSaWebQj0CcFU7rz1rxuQd/kV/4ZJ7g/+6oRifhCLrqia4XprdpEj7xG2zj/IwF85OUSubwu2S/tXM7qxSLmcnqIezTLXz2nmyqBXpoFZkXdUE10uzNRRtn8jr8OY4O0uWMBfTgL9IekTSfZKWSPLpozpRD2eZtDbytp+Wxa0oJ7Ui60rrY96MdCOHdlZ1Tp5yx6k1WZTC0cBE4CjgncA74nenYIrKzJWXtDbS9iXJm5bFLbRv6sTkJDtpGc7yZmubOnF4Yqa2Uh+TsqUdf+joxORDacEA0/rfqOvrOHnIohRON7MV5S/g9FoL1m6EImJC/sxceUnLpBXaBwTlTc2WlsCa515iSy/vtS1bjTXPvRQ8p3vs8MR2QtnaxncNSXcES8iW9uja59naq7i0HRqv02cekBgsr2Q0bsT1dZw8ZDE032VmB5dtdwBLzGy/WgtXoh0MzX0to1M1zmshI2xeqslbUKRTXchonEZfu75Oa1CooVnSHOArQKekZ0vFwMvAOVVL6STS16IvVuO8VpR5uJoMZ0U61VWj2Pra9XXal6BSMLMzgDMknWFmc6qpXNJy4DmijG2vmFl3nIvhYqL0nsuB95pZckD9NqJeGZ2KWuJYjfNaMz4pVONUV8nIneRU1xczdvly2PYkS+jsqhRCGdPM7MCyR5jZwA1mNgm4Id5ue+phcCwyk1co81koW9q0KV1M6BqUuG/3AR25yid0DQo6g03oGpQ4jqFsadOmdAXHPs3QHaIUXK+kgEpOdeNGdPYpg7JnfWtfshiai+YY4IL48wXAzAbI0HSkGXuLoshMXqHMZ6FsaTc9tJZlazcm7nvh5a2JxtkXXu5t5o1YtnZjMFvbsrUbE8dx+brkJ4GbHlobHPtfn3J4MMNaiPkLVyWW375sfZ8yKHvWt/aloqF5pyqXHgXWE63t+LmZnSNpg5kNLTtmvZnt8DdO0qnAqQBjxow5ZMWKFTWTs10YP/vqxHl9AY/mjEY6bvbVuY4XOy7wKScpGmreNtLqKrLvaaTJnDfiayOp13g5taUaQ3PFJwVJEyUNiD8fKem0OEBeFqbGK5eOBj4p6Q1ZBTOzc8ys28y6u7paJyl2I2mko1Q1jl3VOMI12kksr1zNijvVtS9Zpo9+C2yR9CrgPGA8cFGWys3s8fh9DXAF8DrgSUn7AMTva6qQu+lJclRqdPuzZkwOOnDNXbCEiXOuYdzsq5k45xrmLtgWEC9pX2i+fdJeg4NtpDl25S3Pe069nMTyygWN/64k4U517UsWpbDVzF4B3g38yMw+B+xT6SRJgyXtVvoMvAW4H7gKOCk+7CTgymoEb2YabaQLtb9oxdOJDlyXLlqZaBydu2BJ0HA6vmvIDoph6sThfHLapKCTWMixKzRHv3DZumB599jhid7GISe5ethsiNtP8s4OydXo70qIeo2X03xkcV5bCPwI+CrwTjN7VNL9ZrZ/hfMmED0dQLT09SIz+46kEcAlwBhgJXCcme0Yt7iMvua81mhHpbyOZSHSopFWG/UzD2nz86MCSzwb7QyWt/+N/q44rU2toqR+CPg48J1YIYwHLqx0kpktI0rj2bt8HTA9j5B9jUY7KuV17AqRdnyRWcmqodFjHKKoDGuN7ofTvlRUCmb2AHBa2fajwLxaCtXXabSjUl7HshCVnhTytF3qe8ghKqk8jbR2Gul0lSZXqzi1Oa1N0KZQCpEdetVTyL5Go41006Ykr9Y6bMKw3E5aRRp0Q/PncxcsSSzfe7ddE9uYtNfgYB/Hjehs6Bx9SK5Bu/ZLtM2EnABD9ThOrUkzNJdCZF8bv06IX9cAl9VetL5Lo410Nz20NrF8+bpNuZ208hqH0/oecoiav3BVYvkuHR07hMKetNdgrv/8kcE+3r5sfUOdrkJyhZz9QuWhehyn1mQxNN9mZlMrldWSvmZobjTN6ngUkitEmrxF1lUkeeUK0ehr5bQGNXFeAwZLOqKskdcDyZlMnKagWR2P8mY4S5O3yLqKpKh2Gn2tnPYli1L4MPDfkpbHYSt+Gpc5KTTSIalom0bevoSOD8l1/KGjczvVpdVVTd+Lul55g+tNnTjcncScpiJ19VGcUOeNZvZaSbsTTTc9Ux/R+i4lg2ppbrs8y1Y97AqlNopYgZO3L1mOT1pldPGdvQLJxU51tz2yzYWlZJwFemwaSX3sHjs8V9+LvF5pY5+0+uj0mQd4iGqnqchiU7jZzI6sjzjJ9DWbQis5JNXDGSt0TohqMp+l0UrXy3HKqZXz2m2SziJKjNOzVMLM7sopX9vQSg5J9XDGyjsuRSTqydJ+X7xejrOzZFEKr4/fv1VWZoD/hQrQ17KopZHXSayavofOCVF0xFF3IHOcbWTJvDYt4eUKIYW+lkUtjVBfpk3pSmx/2pSu3H0POWr19lEokRZxtBpC7bsDmdOOZMmnsIekH0haFL/+U9Ie9RCur9LXsqilEerLTQ+tTWz/pofW5u57yFFrYyAjW1rms2oIte8OZE47kmX66H+JQl6/N97+IPAL4NhaCdUKzDxoVE1XkNRzHjypL5+7+J5g+3n7ntaXkld1LXGbguNsI4tSmGhm7ynb/qake2okj5ORSvPgoeWP9Wo/RF47RJrdpCibitsUHGcbWZzXNvXyaJ4K+F+oBpNmtwglxinPplbL9kOE7CDjRiTffNOC2xVpU2l0AEPHaSay+CkcCFwA7EEUkuVp4CQzq1uk1L7mp1AvQv+UJ865JldinKLbD5E3+U+ofFT8D75I3wJ3IHNakZr4KZjZPUDJoxkze7Y68ZyiCc3dh9bxF72+vyjbQV55i/R5KFFrG5Dj9BWyrD56RNKvgQ8A+9ZeJGdnCa3jL3p9f17yBrFLC27XrEH/HKevk8WmsB/wc2AE8H1JyyRdUeEcp4GkJcZpxkB9oSB2acHt3A7gOLUhy+qjLcDm+H0r8CSwppZCOTtHaZVR79VH3WOHN22gvlAQu0rB7dwO4DjFksXQvBFYAvwA+IOZrauHYOW4obkYPPCb47QXtUqyczxwC/AJ4DeSvilpejUCOo3FnbQcx6lEltVHVwJXSpoCHA18FvgS4Ba9PkYzO2mFloTWa6moL0l1nIiKSkHSb4EDgb8DtwL/B1hYW7GcWjBrxuTtbArQHMbZUJKbRSue5reLV9fcBtLopEiO00xkMTTPA+4ysy0Vj3SamiIzshVJKLhfyVDeu/zM65bWLbhgo8fGcepNlumjO+shiFMfmtFJK69TW9E2ELe1OM42shiaHaem5HVqK9oG4o5wjrONoFKIA98haUD9xHFanSTnubxObdXaQEKOe+4I5zjbSHtS+K/4/S/1EMRpfUKRTYHExDynzzygsGRFaVFV65EUyXH6CkHnNUm3Aw8CbwMu7r3fzE6rrWjbcOe11qCRznPuuOe0I0VHSX0H8CbgKGDxzgjmONBYg64bkx0nG0GlYGZPEXkwP2hm99ZRJqdFaaTzXDM77jlOM5Fl9dE6SVdIWiPpSUm/leQhtJ3cNNKg68Zkx8lGFqXwC+AqYCQwCvhdXOY4uWikQdeNyY6TjSxRUu81s9f2KrvHzA6spWDluKHZcRwnP7WKkrpW0omSOuLXiUDdw2c7juM4tSdL7KMPA2cBPwQM+HNclglJHcAiYLWZvUPScKIlruOA5cB7zWx9PrGbH4+66ThOX6Tik4KZrTSzd5lZl5ntZWYzzWxFjjY+Q+TvUGI2cIOZTQJuiLdbijRHKcdxnGamprGP4lVKbwfOLSs+Brgg/nwBMLOWMjSCtKibjuM4zUytA+L9iCghz9aysr3N7AmA+H2vpBMlnSppkaRFa9eurbGYxeKOUo7j9FVqphQkvQNYY2ZVeUOb2Tlm1m1m3V1dXQVLV1s86qbjOH2VikpB0tyyz3kipk4F3iVpOfAb4ChJFwJPStonrm8fYE0uifsA7ijlOE5fJS109pckHQ78e1lx5oipZjbHzPY1s3HA+4EbzexEIke4k+LDTgKuzC11k+OOUo7j9FXSlqQuBY4DJki6lWgF0QhJk81sZyym84BLJH0EWBm30XI0Y4Yzx3GcSqQphfXAV4Aj49c/AzOA2bFieH3WRszsZuDm+PM6YHpV0jqO4zg1JU0pvBX4OjAR+AFwL/CCmX2oHoI5juM49SdoUzCzr5jZdCKv4wuJFEiXpD9J+l2d5HMcx3HqSJYwF9eZ2Z3AnZL+w8yOkLRnrQVzHMdx6k+WMBdfKts8OS57qlYCOY7jOI0jl/OaZ2BzHMdpbWod5sJxHMfpQ7hScBzHcXpwpeA4juP04ErBcRzH6cGVguM4jtODKwXHcRynB1cKjuM4Tg+uFBzHcZweXCk4juM4PbhScBzHcXpwpeA4juP04ErBcRzH6cGVguM4jtNDlnwKfZIFd6/mzOuW8viGTYwc2smsGZM9Z7LjOE4FWlIpLLh7NXMuX8KmzVsAWL1hE3MuXwLgisFxHCeFlpw+OvO6pT0KocSmzVs487qlDZLIcRynb9CSSuHxDZtylTuO4zgRLakURg7tzFXuOI7jRLSkUpg1YzKd/Tu2K+vs38GsGZMbJJHjOE7foCUNzSVjsq8+chzHyUdLKgWIFIMrAcdxnHy05PSR4ziOUx2uFBzHcZweXCk4juM4PbhScBzHcXpwpeA4juP0IDNrtAwVkbQWWFHl6XsCTxUoTl+jnfvvfW9f2rn/5X0fa2ZdeU7uE0phZ5C0yMy6Gy1Ho2jn/nvf27Pv0N7939m++/SR4ziO04MrBcdxHKeHdlAK5zRagAbTzv33vrcv7dz/nep7y9sUHMdxnOy0w5OC4ziOkxFXCo7jOE4PLacUJA2VdJmkhyQ9KOlwScMlXS/p4fh9WKPlLBpJkyXdU/Z6VtJn26HvAJI+J+mvku6XNF/SwHbpO4Ckz8R9/6ukz8ZlLdl/Sf8raY2k+8vKgn2VNEfS3yUtlTSjMVIXR6D/x8XXfquk7l7H5+p/yykF4MfAtWY2BXgt8CAwG7jBzCYBN8TbLYWZLTWzA83sQOAQYCNwBW3Qd0mjgNOAbjPbH+gA3k8b9B1A0v7AKcDriL7z75A0idbt//nAW3uVJfZV0n5E34XXxOf8VFIHfZvz2bH/9wPHAreUF1bT/5ZSCpJ2B94AnAdgZi+b2QbgGOCC+LALgJmNkK+OTAceMbMVtE/fdwE6Je0CDAIep336/s/A7Wa20cxeAf4IvJsW7b+Z3QI83as41NdjgN+Y2Utm9ijwdyLl2WdJ6r+ZPWhmSxMOz93/llIKwARgLfALSXdLOlfSYGBvM3sCIH7fq5FC1oH3A/Pjzy3fdzNbDXwfWAk8ATxjZr+nDfoecz/wBkkjJA0C3gaMpn36D+G+jgJWlR33WFzWLuTuf6sphV2Ag4GfmdlBwAu0ziNzJiTtCrwLuLTRstSLeP74GGA8MBIYLOnExkpVP8zsQeD/AtcD1wL3Aq80VKjmQQll7bQOP3f/W00pPAY8ZmYL4+3LiJTEk5L2AYjf1zRIvnpwNHCXmT0Zb7dD398EPGpma81sM3A58Hrao+8AmNl5Znawmb2BaGrhYdqo/4T7+hjRU1OJfYmmFtuF3P1vKaVgZv8AVkmaHBdNBx4ArgJOistOAq5sgHj14ni2TR1Be/R9JXCYpEGSRHTdH6Q9+g6ApL3i9zFEBsf5tFH/Cff1KuD9kgZIGg9MAu5ogHyNIn//zaylXsCBwCLgPmABMAwYQbQi4eH4fXij5axR3wcB64A9ysrape/fBB4iml//FTCgXfoe9/9Woj9A9wLTW/naEym8J4DNRP+EP5LWV+CrwCPAUuDoRstfo/6/O/78EvAkcF21/fcwF47jOE4PLTV95DiO4+wcrhQcx3GcHlwpOI7jOD24UnAcx3F6cKXgOI7j9OBKwWk6JJ0h6UhJMyXl8kiX1CVpYRzm5N9qINvJks4qut5aIOl8Sf/eaDmcvoUrBacZORRYCLyRaP19HqYDD5nZQWaW99w+SwtE/nSaBFcKTtMg6UxJ9wH/CvwF+CjwM0lfSzh2rKQbJN0Xv4+RdCDwPeBtcU6Jzl7nzJP0QHzO9+Oy8yWdLelWSX+T9I64fKCkX0haEj91TEuQ4e2S/iJpT0lviT/fJelSSUPiY5ZL+mZcvkTSlIR6rpH0L/Hnu0v9lfRtSR9VxJmK8iUskfS+eP+Rkm6SdBGwJD7urLiPV1MWAC+p746TSKO98/zlr/IXUVjfnwD9gdtSjvsdcFL8+cPAgvjzycBZCccPJ/LoLDlsDo3fzycKItePKATAY8BA4AvAL+JjphCF0hhYqp/Ig/RWIo/5PYni2A+Oj/8y8LX483Lg0/HnTwDnJsg2G/gksDtwJ7E3KnATMBl4D1Gwuw5g71iWfYAjiYI+jo+PP7bsuJHABuDfQ333l7+SXv6k4DQbBwH3EN2IH0g57nDgovjzr4AjKtT7LPAicK6kY4mSEJW4xMy2mtnDwLK47SPiejGzh4AVwKvj46cR3fjfbmbrgcOA/YDbJN1DFHtnbFn9l8fvi4FxCbLdSpQH5AjgamBIHAJ7nEUx8o8A5pvZFosCHf6R6GkK4A6L4uQT11E67nHgxgx9d5zt2KXRAjgOQDz1cz5RFMeniOI4Kb7JHm5mmypUkRqvxcxekfQ6IpvD+4FPAUcFzjWSQw6XWEaUu+PVRHG2BFxvZscHjn8pft9C8m/uTqA7rvd6oiePU4iUCBVkeSFB9u0L0vvuONvhTwpOU2Bm91iUSvRvRP+6bwRmWJRiNEkh/JnoBgdwAvCntPrjOf49zOwa4LNEgRNLHCepn6SJRDf7pUTTQSfE574aGBOXQ/TUcCzwS0mvAW4Hpkp6VXz8oPicrH1/mSgRynvjum4Fvsg2I/stwPskdUjqInoiSIp0eQtRRMyOOHz0tAx9d5zt8CcFp2mIb3jrzWyrpClmljZ9dBrwv5JmEWXb+1CF6ncDrpQ0kOif9+fK9i0lmpLZG/i4mb0o6afA2ZKWECWsOdnMXooic0c5sSWdQJTM6J1Etob5kgbEdc4lUnBZuZUouulGSbcSPTGVlMIVRNNl9xI9CXzJzP6RYLS+gugJYEnc9h8z9N1xtsOjpDptjaTzgf9nZpc1WhbHaQZ8+shxHMfpwZ8UHMdxnB78ScFxHMfpwZWC4ziO04MrBcdxHKcHVwqO4zhOD64UHMdxnB7+fw3bq5A6Kp1AAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
    " + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plt.scatter(output.truth, output.prediction)\n", + "plt.xlabel('# of spoken words')\n", + "plt.ylabel('# of words transcribed correctly')\n", + "plt.title('No. of spoken words vs No. of words transcribed correctly')\n", + "word_match_plot = plot_to_str()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Analysing model perfomance" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAr8AAAF2CAYAAABnFvAPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAcZUlEQVR4nO3de7htZV0v8O9PwVuiomwVEaTykpdylzu8daG0jqKGmjdUxJOJ2fEc7WBlZml5P3lBMzVMhLyQ5iUVsUSStFRy4yEvB28ZCoKwEVS8I/7OH2NsXXvtta9rLfbevJ/P88xnjTneMcf4zTHGms93vvOdY1Z3BwAARnC1XV0AAABcWYRfAACGIfwCADAM4RcAgGEIvwAADEP4BQBgGMIvsGqqqqvqliu8ztOr6rdXcp1ctVXVM6rqdbu6DmD3IPzCHqKqzqmq71XVfovmnzWHzIN3UV0/XlU/qKqX74rtb81yg/L8+O9U1TcW3N65kjUCcOUSfmHP8l9Jjth4p6p+Osm1d105SZJHJbk0ycOq6pq7uJbV8ITuvu6C2/2WWqiq9tqeeVuzo8vv4LqvvlrrHslqHiPgyiH8wp7ltZnC5kZHJfnbhQtU1TWr6gVV9cWqurCqXllV157b9q2qk6tqQ1VdOk/ffMFjT6+qZ1bVv1XVZVX1nsU9zUt4VJKnJbk8yVLB8LCq+nxVXVxVf1FVV5u3dcuq+peq+trc9sYFddytqj4yt32kqu621IYXf5xdVQfPveB7VdWzk/xikpfNPbYvm5f5qao6taouqapPV9VDtvH8llRVh1bVeVX1h1X15SSvmet5c1W9rqq+nuTRVXWzqnrHvL3PVdVjF9W/yfJLbOc+VfV/q+rrVXVuVT1jUfsvVNUHq+qrc/uj5/knVNUrquqUqvpmkl+pqtvOx/irVfXJqvqNBes5rKr+33zcv1RVT57n7zefJ1+dn8MHNh7DRXVUVb24qi6aj9vHquoOc9vWzslTquqFC9bzxqo6fgv7/JCqWj/viwur6kULj8WiZc+pqnsumHWted2XVdVHq+qOC5b9uXkfX1ZVfz8v96ytHOdrVtWxVXX+fDu25jd+VfXoqvrXRbX8cPjPfFxeOZ+Dl9X0P3CLbe1DYOUIv7Bn+XCS680h5upJHppk8VjG5ye5dZK1SW6Z5IAkfzq3XS3Ja5LcIslBSb6d5GWLHv/wJP89yY2TXCPJk7dUTFX9YpKbJ/m7JG/KpsF8owckWZfk55IcnuS35vnPTPKeJPvO6/jLeZ03TPKuJC9NcqMkL0ryrqq60ZbqWEp3/3GSD+RHPbdPqKofS3JqkjfMz++IJC+vqtvvyLoXuGmSG2ban0fP8w5P8uYkN0jy+iQnJTkvyc2SPCjJc6rqHgvWsXj5xb6Zab/eIMl9kjy+qu6fJFV1UJJ3Z9p3azId87MWPPbhSZ6dZJ8kZyR5Z6Z9fuMk/zPJ66vqNvOyr07yuO7eJ8kdkvzzPP+Yuf41SW6S5KlJeok6fz3JL2U6926Q6dz8yty2tXPyt5IcWVW/WlWPSPLzSZ64xPqT5CVJXtLd10vyk5nOue11eJK/z3S83pDkH6pq76q6RpK3JTlhbjsp0zm70OLj/MdJ7jI/nzsmOSTTG8Dt9YhM5/9+mY7XxuO+tX0IrBDhF/Y8G3t/fy3Jp5J8aWNDVVWSxyb5ve6+pLsvS/KcJA9Lku7+Sne/pbu/Nbc9O8kvL1r/a7r7M9397UzhYu1Wajkqybu7+9JMgeLeVXXjRcs8f67li0mOzY+GbVyeKUzcrLu/090be8vuk+Sz3f3a7v5+d580P88lhxvsoPsmOae7XzOv+6NJ3pIplG7JS+dez423Zy5o+0GSp3f3d+f9lSQf6u5/6O4fZAo3v5DkD+fneFaSv0ly5IJ1/HD5Bev4oe4+vbs/Prd/LFM423jMHpHkvd19UndfPh/fsxY8/O3d/W9zLWuTXDfJ87r7e939z0lOzqbH43ZVdb3uvnTeNxvn75/kFvM2PtDdS4XfyzOF7J9KUt19dndfsB3n5JeT/E6SEzOF20fNyyzl8iS3rKr9uvsb3f3hLSy3lDO7+83dfXmmN1TXyhRg75JkryQvnZ/fW5P8+6LHLj7Oj0jy5919UXdvSPJn2fSYbsu7uvv93f3dTEH6rlV1YLawD3dgvcB2EH5hz/PaTD16j86iIQ+Zeueuk+TMjWEtyT/O81NV16mqv66qL9T0Mfv7k9ygNh0P+uUF09/KFJg2M39s/eDMvVbd/aEkX5xrW+jcBdNfyNQDmiR/kKSS/Pv8EfzGHuGbzctl0eMOWKqOHXSLJHdeGGYzBZmbbuUx/6u7b7Dg9icL2jZ093cWLb/w+d4sySWLwtzi57Jw+c1U1Z2r6n01DVX5WqaguHEoyoFJ/nMrD19cy7lzEF6qlt9McliSL8wfxd91nv8XST6X5D01DV95ylIbmsP0y5L8VZILq+q4qrpetnFOzk5OcvUkn17wJmgpj8nUK/qpmobD3Hcryy72w30x74ONvfE3S/KlRYF+8TFZfJwXn6MLz+sdreUbSS7J9CZwS/sQWEHCL+xhuvsLmb74dliSty5qvjjTUIbbLwhr1+/ujQH2mCS3SXLn+aPjX5rn106U8oAk18s0bODL83jIA7L50IcDF0wflOT8+Xl8ubsf2903S/K4eT23nNtvsWgdB2VBD/cC38wUrDZaHGIX91Cem+RfFoXZ63b347f6TLdsqR7QhfPOT3LDqtpnwbzFz2WpdSz0hiTvSHJgd18/ySvzo+N1bqaP/7envvOTHFibjtf9YS3d/ZHuPjzTkIh/yDykoLsv6+5juvsnMvW+/+9FwzZ+tLHul3b3nZLcPlNI/f1s+5xMpk8gzk6yf1UdsdmKf7T+z3b3EXONz0/y5nkoyybnwfxmbs2ihx+4oP1qmYbanJ/kgiQHzD3Umy27cdOL7i8+R394Xi9Ry1JvrBbWct1MQyo2/l8stQ+BFST8wp7pMUl+tbu/uXDm3KP1qiQv3jj8oKoOqKr/Ni+yT6Yg8tV5bO3Tl1HDUUmOT/LTmT5SX5vk7knW1nQVio1+v6Yv2h2YaSznG+e6Hlw/+rLdpZkCxhVJTkly66p6eE1fXHtokttl6h1c7Kwkv1RVB1XV9ZP80aL2C5P8xIL7J8/rPnIe77l3Vf18Vd1253bB1nX3uUk+mOS5VXWtqvqZTMduqbG9W7JPpt7j71TVIdm0Z/31Se5ZVQ+Z99WNqmrtFtZzRqZg9gfz8z40U5j9u6q6RlU9oqquPw8L+HqmY5Gqum9NX06sBfOvWLzyeT/euar2nrfznSRXbOucrKpfyjTG/FHz7S+rasle/qp6ZFWtmdf51Xn2FUk+k+kLbfeZt/+0JIuvPHKnqnpgTVdreFKS72YaQ/+heR1PmPfh4ZnG8G7NSUmeVlVravpC6J/mR2Pv/yPJ7atqbVVdK8kzlnj8YTV9UfEamcb+ntHd525pH26jFmAHCb+wB+ru/+zu9Vto/sNMH1N/eB7a8N5Mvb3JNOb22pl64z6c6ePnHTaHk3skOXbuwd14O3Ne51ELFn97kjMzBdV3ZfpiVTJ9semMqvpGpp7NJ3b3f3X3VzKNzT0m05d9/iDJfbv74sV1dPepmcL0x+ZtLA7IL0nyoJqubPHSefjBr2cab3p+piEez8/mQWmhjVeL2Hg7c3v20QJHJDl43t7bMo0dPXUHHv+7Sf68qi7LFLJ++CWvnsZRH5ZpX12SaR/fcYl1pLu/l+Q3ktw70/F/eabxtZ+aFzkyyTnzOfM7SR45z79VpnPoG5mC4su7+/QlNnG9TCH30kzDAL6S5AVz25Ln5PyR/t9m+lLil+YhD6/OdEWFpT6NuFeST87nzEuSPGweS/21eT/9Taae7G9mGtaw0NszfYHs0vm5PnAe4/u9JA/M9Kbkq/PzPjlTON6SZyVZn+m8+3iSj87z0t2fSfLn83P8bJKlhnG8IdMbz0uS3CnT0Jtk6/sQWCG19PcWAGBMVXVGkld292tWYd0nJDmvu3fk6hDACtLzC8DQquqXq+qm87CHo5L8THbyUxFg9+eXagAY3W0yDSe5bqarZzzIJcbgqsuwBwAAhmHYAwAAwxB+AQAYxpU65ne//fbrgw8++MrcJAAAAzrzzDMv7u7FP3hz5Ybfgw8+OOvXb+nSpAAAsDKq6gtLzTfsAQCAYQi/AAAMQ/gFAGAYwi8AAMMQfgEAGIbwCwDAMIRfAACGsc3wW1XXqqp/r6r/qKpPVtWfzfNvWFWnVtVn57/7rn65AACw87an5/e7SX61u++YZG2Se1XVXZI8Jclp3X2rJKfN9wEAYLe1zfDbk2/Md/eeb53k8CQnzvNPTHL/1SgQAABWynaN+a2qq1fVWUkuSnJqd5+R5CbdfUGSzH9vvIXHHl1V66tq/YYNG1aobAAA2HHbFX67+4ruXpvk5kkOqao7bO8Guvu47l7X3evWrFmzk2UCAMDy7dDVHrr7q0lOT3KvJBdW1f5JMv+9aKWLAwCAlbQ9V3tYU1U3mKevneSeST6V5B1JjpoXOyrJ21epRlg1hx56aA499NBdXQYAcCXZazuW2T/JiVV19Uxh+U3dfXJVfSjJm6rqMUm+mOTBq1gnAAAs2zbDb3d/LMnPLjH/K0nusRpFAQDAavALbwAADEP4BQBgGMIvAADDEH4BABiG8AsAwDCEXwBgp7leOnsa4RcAgGEIvwAADEP4BQBgGMIvAADDEH4BABiG8AsAwDCEXwAAhiH8AgAwDOEXAIBhCL8AAAxD+AUAYBjCLwAAwxB+AQAYhvALAMAwhF8AAIYh/AIAMAzhFwCAYQi/AAAMQ/gFAGAYe12pW/v0p5NDD9103kMekvzu7ybf+lZy2GGbP+bRj55uF1+cPOhBm7c//vHJQx+anHtucuSRm7cfc0xyv/tN237c4zZvf9rTknveMznrrORJT9q8/TnPSe52t+SDH0ye+tTN2489Nlm7Nnnve5NnPWvz9r/+6+Q2t0ne+c7khS/cvP21r00OPDB54xuTV7xi8/Y3vznZb7/khBOm22KnnJJc5zrJy1+evOlNm7effvr09wUvSE4+edO2a187efe7p+lnPjM57bRN2290o+Qtb5mm/+iPkg99aNP2m988ed3rpuknPWnahwvd+tbJccdN00cfnXzmM5u2r1077b8keeQjk/PO27T9rndNnvvcafo3fzP5ylc2bb/HPZI/+ZNp+t73Tr797U3b73vf5MlPnqYXn3fJdO4lueYVVyzd7txz7iWrd+553XPuXUXOvWM37oNDD3XuOfem6d3xdW8BPb8AAAyjuvtK29i6det6/fr1V9r2YFsOnd+dnr7x3ToAO8TrKLurqjqzu9ctnq/nFwCAYQi/AAAMQ/gFAGAYwi8AAMMQfgEAGIbwCwDAMIRfAACGceX+wtsucvBT3rWrS2A39eXPT79g4xxhKec87z67ugQAVpieXwAAhjFEzy8Ay+cTEpbiEzS2Znf8BE3PLwAAwxB+AQAYhvALAMAwhF8AAIYh/AIAMIxtht+qOrCq3ldVZ1fVJ6vqifP8Z1TVl6rqrPl22OqXCwAAO297LnX2/STHdPdHq2qfJGdW1alz24u7+wWrVx4AAKycbYbf7r4gyQXz9GVVdXaSA1a7MAAAWGk7NOa3qg5O8rNJzphnPaGqPlZVx1fVvitdHAAArKTtDr9Vdd0kb0nypO7+epJXJPnJJGsz9Qy/cAuPO7qq1lfV+g0bNiy/YgAA2EnbFX6rau9Mwff13f3WJOnuC7v7iu7+QZJXJTlkqcd293Hdva67161Zs2al6gYAgB22PVd7qCSvTnJ2d79owfz9Fyz2gCSfWPnyAABg5WzP1R7unuTIJB+vqrPmeU9NckRVrU3SSc5J8rhVqA8AAFbM9lzt4V+T1BJNp6x8OQAAsHr8whsAAMMQfgEAGIbwCwDAMIRfAACGIfwCADAM4RcAgGEIvwAADEP4BQBgGMIvAADDEH4BABiG8AsAwDCEXwAAhrHXri4AdqWbPvx5u7oEgD2a11H2NHp+AQAYhvALAMAwhF8AAIYh/AIAMAzhFwCAYQi/AAAMQ/gFAGAYwi8AAMMQfgEAGIbwCwDAMIRfAACGIfwCADAM4RcAgGEIvwAADEP4BQBgGMIvAADDEH4BABiG8AsAwDCEXwAAhiH8AgAwDOEXAIBhCL8AAAxD+AUAYBjCLwAAwxB+AQAYhvALAMAwhF8AAIYh/AIAMAzhFwCAYQi/AAAMY5vht6oOrKr3VdXZVfXJqnriPP+GVXVqVX12/rvv6pcLAAA7b3t6fr+f5Jjuvm2SuyT5H1V1uyRPSXJad98qyWnzfQAA2G1tM/x29wXd/dF5+rIkZyc5IMnhSU6cFzsxyf1XqUYAAFgROzTmt6oOTvKzSc5IcpPuviCZAnKSG694dQAAsIK2O/xW1XWTvCXJk7r76zvwuKOran1Vrd+wYcPO1AgAACtiu8JvVe2dKfi+vrvfOs++sKr2n9v3T3LRUo/t7uO6e113r1uzZs1K1AwAADtle672UEleneTs7n7RgqZ3JDlqnj4qydtXvjwAAFg5e23HMndPcmSSj1fVWfO8pyZ5XpI3VdVjknwxyYNXpUIAAFgh2wy/3f2vSWoLzfdY2XIAAGD1+IU3AACGIfwCADAM4RcAgGEIvwAADEP4BQBgGMIvAADDEH4BABiG8AsAwDCEXwAAhiH8AgAwDOEXAIBhCL8AAAxD+AUAYBjCLwAAwxB+AQAYhvALAMAwhF8AAIYh/AIAMAzhFwCAYQi/AAAMQ/gFAGAYwi8AAMMQfgEAGIbwCwDAMIRfAACGIfwCADAM4RcAgGEIvwAADEP4BQBgGMIvAADDEH4BABiG8AsAwDCEXwAAhiH8AgAwDOEXAIBhCL8AAAxD+AUAYBjCLwAAwxB+AQAYhvALAMAwhF8AAIYh/AIAMAzhFwCAYQi/AAAMY5vht6qOr6qLquoTC+Y9o6q+VFVnzbfDVrdMAABYvu3p+T0hyb2WmP/i7l47305Z2bIAAGDlbTP8dvf7k1xyJdQCAACrajljfp9QVR+bh0Xsu6WFquroqlpfVes3bNiwjM0BAMDy7Gz4fUWSn0yyNskFSV64pQW7+7juXtfd69asWbOTmwMAgOXbqfDb3Rd29xXd/YMkr0pyyMqWBQAAK2+nwm9V7b/g7gOSfGJLywIAwO5ir20tUFUnJTk0yX5VdV6Spyc5tKrWJukk5yR53OqVCAAAK2Ob4be7j1hi9qtXoRYAAFhVfuENAIBhCL8AAAxD+AUAYBjCLwAAwxB+AQAYhvALAMAwhF8AAIYh/AIAMAzhFwCAYQi/AAAMQ/gFAGAYwi8AAMMQfgEAGIbwCwDAMIRfAACGIfwCADAM4RcAgGEIvwAADEP4BQBgGMIvAADDEH4BABiG8AsAwDCEXwAAhiH8AgAwDOEXAIBhCL8AAAxD+AUAYBjCLwAAwxB+AQAYhvALAMAwhF8AAIYh/AIAMAzhFwCAYQi/AAAMQ/gFAGAYwi8AAMMQfgEAGIbwCwDAMIRfAACGIfwCADAM4RcAgGEIvwAADEP4BQBgGNsMv1V1fFVdVFWfWDDvhlV1alV9dv677+qWCQAAy7c9Pb8nJLnXonlPSXJad98qyWnzfQAA2K1tM/x29/uTXLJo9uFJTpynT0xy/5UtCwAAVt7Ojvm9SXdfkCTz3xtvacGqOrqq1lfV+g0bNuzk5gAAYPlW/Qtv3X1cd6/r7nVr1qxZ7c0BAMAW7Wz4vbCq9k+S+e9FK1cSAACsjp0Nv+9IctQ8fVSSt69MOQAAsHq251JnJyX5UJLbVNV5VfWYJM9L8mtV9dkkvzbfBwCA3dpe21qgu4/YQtM9VrgWAABYVX7hDQCAYQi/AAAMQ/gFAGAYwi8AAMMQfgEAGIbwCwDAMIRfAACGIfwCADAM4RcAgGEIvwAADEP4BQBgGMIvAADDEH4BABiG8AsAwDCEXwAAhiH8AgAwDOEXAIBhCL8AAAxD+AUAYBjCLwAAwxB+AQAYhvALAMAwhF8AAIYh/AIAMAzhFwCAYQi/AAAMQ/gFAGAYwi8AAMMQfgEAGIbwCwDAMIRfAACGIfwCADAM4RcAgGEIvwAADEP4BQBgGMIvAADDEH4BABiG8AsAwDCEXwAAhiH8AgAwDOEXAIBhCL8AAAxD+AUAYBh7LefBVXVOksuSXJHk+929biWKAgCA1bCs8Dv7le6+eAXWAwAAq8qwBwAAhrHc8NtJ3lNVZ1bV0StREAAArJblDnu4e3efX1U3TnJqVX2qu9+/cIE5FB+dJAcddNAyNwcAADtvWT2/3X3+/PeiJG9LcsgSyxzX3eu6e92aNWuWszkAAFiWnQ6/VfVjVbXPxukkv57kEytVGAAArLTlDHu4SZK3VdXG9byhu/9xRaoCAIBVsNPht7s/n+SOK1gLAACsKpc6AwBgGMIvAADDEH4BABiG8AsAwDCEXwAAhiH8AgAwDOEXAIBhCL8AAAxD+AUAYBjCLwAAwxB+AQAYhvALAMAwhF8AAIYh/AIAMAzhFwCAYQi/AAAMQ/gFAGAYwi8AAMMQfgEAGIbwCwDAMIRfAACGIfwCADAM4RcAgGEIvwAADEP4BQBgGMIvAADDEH4BABiG8AsAwDCEXwAAhiH8AgAwDOEXAIBhCL8AAAxD+AUAYBjCLwAAwxB+AQAYhvALAMAwhF8AAIYh/AIAMAzhFwCAYQi/AAAMQ/gFAGAYwi8AAMMQfgEAGMaywm9V3auqPl1Vn6uqp6xUUQAAsBp2OvxW1dWT/FWSeye5XZIjqup2K1UYAACstOX0/B6S5HPd/fnu/l6Sv0ty+MqUBQAAK2854feAJOcuuH/ePA8AAHZLey3jsbXEvN5soaqjkxw93/1GVX16GduE1bBfkot3dRHsfur5u7oC2GN4HWVJu/h19BZLzVxO+D0vyYEL7t88yfmLF+ru45Ict4ztwKqqqvXdvW5X1wGwp/I6yp5kOcMePpLkVlX141V1jSQPS/KOlSkLAABW3k73/Hb396vqCUn+KcnVkxzf3Z9cscoAAGCFLWfYQ7r7lCSnrFAtsKsYlgOwPF5H2WNU92bfUQMAgKskP28MAMAwhF8AAIYh/MI2VNU5VfXxqjprvt1tV9cEsKeYX0M/sGjeWVX1iV1VE2Nb1hfeYCC/0t0u4A6wc/apqgO7+9yquu2uLoax6fnlKqOqfqyq3lVV/1FVn6iqh1bVnarqX6rqzKr6p6rav6quX1WfrqrbzI87qaoeu6vrB9iVVvk19E1JHjpPH5HkpNV8LrA1wi9XJfdKcn5337G775DkH5P8ZZIHdfedkhyf5Nnd/bUkT0hyQlU9LMm+3f2qbaz7ffPHdGes5hMA2IVW8zX0zUkeOE/fL8k7V+UZwHYw7IGrko8neUFVPT/JyUkuTXKHJKdWVTL9GMsFSdLdp1bVg5P8VZI7bse6DXsArupW8zX0kiSXzmH57CTfWvnyYfsIv1xldPdnqupOSQ5L8twkpyb5ZHffdfGyVXW1JLdN8u0kN0xy3pVZK8Du5kp4DX1jprD86JWqGXaGYQ9cZVTVzZJ8q7tfl+QFSe6cZE1V3XVu37uqbj8v/nuZeh+OSHJ8Ve29K2oG2F1cCa+hb0vyf5L804oXDztAzy9XJT+d5C+q6gdJLk/y+CTfT/LSqrp+pvP92Kq6PMlvJzmkuy+rqvcneVqSp++iugF2B6v6GtrdlyV5fpLMwyhgl/DzxgAADMOwBwAAhmHYA8zmy5hdc9HsI7v747uiHoA9iddQ9hSGPQAAMAzDHgAAGIbwCwDAMIRfAACGIfwCADAM4RcAgGH8fxUhV4DqMB3wAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
    " + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAr8AAAF2CAYAAABnFvAPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAh2klEQVR4nO3debhkVXm28fsRUFABGRpk7qg4oGLr1WIUhzbgAKLIJ7Mi5DNCNBhJ8FMkzihinHCCiFEhoAwRjQqoDJGoSJAGGQUcEARBaCZBBWV4vz/2PlB9hj5zd8O6f9dV19m11q6936pd+9RzVq2qk6pCkiRJasHDlnUBkiRJ0tJi+JUkSVIzDL+SJElqhuFXkiRJzTD8SpIkqRmGX0mSJDXD8CtpRiWpJE+Y4W2emeTvZnKbapPPJUmGX2k5luSqJH9Jsvaw9gv6kDl3GdX1V0nuS3LYstj/kkw33PS3vyvJHwYu357JGiVJy47hV1r+/RrYbehKkqcDqyy7cgB4PXArsGuSRyzjWmbDvlX16IHLK0dbKcmKE2lbksmuP8ltrzBb2x5jf7N2X5ZHrd1f6aHC8Cst/46mC5tD9gT+Y3CFJI9I8rEkv0lyQ5J/S7JK37dGkpOSLEpya7+84cBtz0xyUJKzktyR5NThI82jeD3wLuBuYLRguG2SK5PclOSjSR7W7+sJSf4nye/7vuMH6nheknP7vnOTPG+0HSd5X5JjBq7P7UfBV0zyIeAFwGf7EdvP9us8OclpSW5JckWSnce5f6NKsiDJtUnekeR3wJf7er6W5JgktwN7JVk/ybf6/f0yyRuH1b/Y+qPs5xVJfprk9iTXJHnfsP7nJ/lxktv6/r369iOTHJ7klCR/BF6c5Cn9Mb4tyaVJXjWwnW2T/Kw/7r9N8ra+fe3+eXJbfx9+OHQMR6m1kvxDkl8Av+jbPtXXdXuS85K8YGD9FZIcmORX/X7PS7LRZI9Tkr3659gdSX6d5LUDj++oz4+Bmz8+yU/659o3k6w5sP7rk1yd5OYk70737svWYx27cY71kUk+OHB9QZJrB65fleSd/TG4NcmXk6w82WMgaXI8kaTl3/8Cq/UhZgVgF+CYYet8BHgiMA94ArAB8J6+72HAl4FNgI2BO4HPDrv97sDfAusADwfeNlYxfZDZEDgOOIHFg/mQHYD5wLOA7YH/27cfBJwKrNFv4zP9NtcETgY+DawFfAI4OclaY9Uxmqr6F+CHPDByu2+SRwGnAV/t799uwGFJnjqZbQ94LLAm3eO5d9+2PfA14DHAV4BjgWuB9YEdgYOTbDWwjeHrD/dHusf1McArgDcleTVAko2B79A9dnPojvkFA7fdHfgQsCpwDvBtusd8HeAtwFeSPKlf94vAPlW1KvA04L/79v37+ucA6wIHArWEx+TVwHOAzfrr5/Z1rUn3uP/nUKgD/pnuGGwLrEb33PjTZI5Tv+6ngW362p837DEYz+v7/a4P3NNviySbAYcBrwXWA1anO5cGTfZYj+e1wMuAx9Odw+/q2yd7DCRNkOFXenAYGv19CXA58NuhjiQB3gj8U1XdUlV3AAcDuwJU1c1VdWJV/anv+xDwomHb/3JV/byq7qQLtPOWUMuewHeq6la6oLJNknWGrfORvpbfAIfywLSNu+lC4/pVdVdV/ahvfwXwi6o6uqruqapj+/s56nSDSdoOuKqqvtxv+3zgRLqgMpZP9yNuQ5eDBvruA95bVX/uHy+As6vqv6rqPmBt4PnAO/r7eAHw78AeA9u4f/2Bbdyvqs6sqov7/ovoAtbQMXstcHpVHVtVd/fH94KBm3+zqs7qa5kHPBo4pKr+UlX/DZzE4sdjsySrVdWt/WMz1L4esEm/jx9W1ZKC14f7431nX/8xfV33VNXHgUcAQ4H774B3VdUV1bmwqm5m8sfpPuBpSVapquur6tIl1Dfc0VV1SVX9EXg3sHP/h+WOwLer6kdV9Re6PyCH3+/JHuvxfLaqrqmqW+jOzcFjM5ljIGmCDL/Sg8PRdCN6ezFsygPdyNAjgfOGwhrw3b6dJI9M8vn+rdzbgR8Aj8ni80F/N7D8J7rANEK6qRQ70Y9WVtXZwG/62gZdM7B8Nd2oGMDbgQA/6d+CHxoRXr9fj2G3Gz7qNhWbAM8ZDLN0AfKxS7jNP1bVYwYu7x7oW1RVdw1bf/D+rg8M/REyZPh9GVx/hCTPSfL9dFNVfg/8PV3QAtgI+NUSbj68lmv6oDZaLa+hG4G9Ot10lOf27R8Ffgmc2k8tOGBJ9Q6/P0n2T3JZP63gNroR1PHqn/Bx6kPrLnSPy/VJTk7y5HFqHKveq4GV+vrWH+yrqj8BNy/hthM51pOtZehcmewxkDRBhl/pQaCqrqb74Nu2wNeHdd9EN5XhqQNhbfWqGgqw+9ONuj2nqlYDXti3Zwql7ED3VvVhSX6Xbt7rBoyc+rDRwPLGwHX9/fhdVb2xqtYH9um384S+f5Nh29iYgRHuAX+kC/tDhoej4aNj1wD/MyzMPrqq3rTEezq20UbfBtuuA9ZMsupA2/D7Mt4I3leBbwEbVdXqwL/xwPG6hu4t8onUdx2w0bC5ovfXUlXnVtX2dNMM/otu1J+quqOq9q+qx9GNvv/zOG/l37/PflrMO4CdgTWq6jHA7ydQ/6SOU1V9r6peQjc6ejnwhb5rvOcHjHx+3k13Hl1PNx1n6L6sQjcNZ9T7yvjHeiq1DJ0rkz0GkibI8Cs9eLwB+Jt+1Ot+/ajeF4BPDk0/SLJBkpf1q6xKF45v6+fWvncaNewJfAl4Ot1b6vOALYF56b6FYsj/S/dBu42AtwLH93XtlAc+bHcrXZC4FzgFeGKS3dN9cG0XuvmjJ41SwwXAC5NsnGR14J3D+m8AHjdw/aR+23skWam/PDvJU6b2ECxZVV0D/Bj4cJKVk2xOd+xGm9s7llXpRhTvSrIFi4+sfwXYOsnO/WO1VpJ5Y2znHLoA9vb+fi+gC1LHJXl4ktcmWb2q7gZupzsWJNku3YcTM9B+7yRqvwdYBKyY5D10fzAN+XfgoCSbprN5P7d7wscpybpJXtXP/f0z8IeB+i5gyc8PgNcl2SzJI4EPAF+rqnvp5vK+Mt2HLx8OvJ8l/JE4gWN9Ad2HP9dM8lhgv1E28w9JNuzPzQN54FyZzjGQtASGX+lBoqp+VVULx+h+B91bpP/bT204nQfmWB5K99VoN9F9eO67U9l/kg2ArYBD+xHcoct5/Tb3HFj9m8B5dC/+J9N9sArg2cA5Sf5AN7L51qr69cCcz/3p3mZ+O7BdVd00vI6qOo0uIFzU72N4QP4UsGO6T89/un9L+qV0c6Cvo5vi8RG6eahjGfq2iKHLeRN5jAbsBszt9/cNujnCp03i9m8GPpDkDrp5pycMdVQ3j3pbusfqFrrH+BmjbaSft/oqYBu6438Y8PqqurxfZQ/gqv458/fA6/r2TemeQ38AzgYOq6ozJ1j79+g+kPdzurfx72Lxt/Y/0d+fU+lC3ReBVSZ5nB7W3//r+sfgRXSP2USeH9BNIzqy38fKwD/2t72U7kOBx9GNAt8B3EgXsMeypGN9NHAhcFV/f48f5fZf7fuu7C9D3w4xnWMgaQni/HlJkkZK8mjgNmDTqvr1LGz/KuDvqur0md62pLE58itJUi/JK/sPiT4K+BhwMd3IraSHCMOvJEkP2J5uCsN1dFMPdvUrxqSHFqc9SJIkqRmO/EqSJKkZhl9JkiQ1Y8WlubO111675s6duzR3KUmSpAadd955N1XVnOHtSzX8zp07l4ULx/qaUkmSJGlmJLl6tHanPUiSJKkZhl9JkiQ1w/ArSZKkZhh+JUmS1AzDryRJkpph+JUkSVIzDL+SJElqhuFXkiRJzTD8SpIkqRmGX0mSJDXD8CtJkqRmGH4lSZLUDMOvJEmSmmH41ZQtWLCABQsWLOsyJEmSJszwK0mSpGYYfiVJktQMw68kSZKaYfiVJElSMwy/kiRJaobhV5IkSc0w/EqSJKkZhl9JkiQ1w/ArSZKkZhh+JUmS1AzDryRJkpoxbvhNsnKSnyS5MMmlSd7ft6+Z5LQkv+h/rjH75UqSJElTN5GR3z8Df1NVzwDmAS9P8tfAAcAZVbUpcEZ/XZIkSVpujRt+q/OH/upK/aWA7YGj+vajgFfPRoGSJEnSTJnQnN8kKyS5ALgROK2qzgHWrarrAfqf68xalZIkSdIMmFD4rap7q2oesCGwRZKnTXQHSfZOsjDJwkWLFk2xTEmSJGn6JvVtD1V1G3Am8HLghiTrAfQ/bxzjNkdU1fyqmj9nzpzpVStJkiRNw0S+7WFOksf0y6sAWwOXA98C9uxX2xP45izVKEmSJM2IFSewznrAUUlWoAvLJ1TVSUnOBk5I8gbgN8BOs1inJEmSNG3jht+qugh45ijtNwNbzUZRkqTOggULADjzzDOXaR3SQ5XnWHv8D2+SJElqhuFXkiRJzTD8SpIkqRmGX0mSJDXD8CtJkqRmGH4lSZLUDMOvJEmSmmH4lSRJUjMMv5IkSWqG4VeSJEnNMPxKkiSpGYZfSZIkNcPwK0mSpGYYfiVJktQMw68kSZKaYfiVJElSMwy/kiRJaobhV5IkSc0w/EqSJKkZhl9JkiQ1w/ArSZKkZhh+JUmS1AzDryRJkpqx4lLd2xVXwIIFi7ftvDO8+c3wpz/BttuOvM1ee3WXm26CHXcc2f+mN8Euu8A118Aee4zs339/5p71MB5387Uc/L3Pjuj+zPN25ay589jshit5zxlHjOj/1xfuyfkbPoVnXXsZb//BUSP6P7DV3vxs3cex5VUX8JYfHzei/8CX7cuVa23IVr88hzf+5Bsj+v9pu/25frU5bHfZD3jdT08Zefde/U5ufeTq7Hjx6ex48ekj+vfa6X3ctdLKvO78k9nu8h+O6N9190MAeOM5X2erX/1ksb67VnwEe+38fgDectaxbHn1hYv137rKarxphwMBePv/HMmzfnv5Yv373HwNf7vWRsw94GTec/oRbHbjlYv1X7nmBhz48rcAcPB3P8PjbvntYv0/W+dxfGDrvQH45Lc/xnp33LRY//kbPJl/fdFeABz+jYNZ487bF+s/a5Nn8JktdwPgyBPey8r3/Hmx/jMevwVfeM7/AeC4rx4w4rE56ckv4JhnvYKV776LI//zfSP6v/b0rfna07dmjT/9nsP/68Mj+o955rac9JQXst7ti/jkSR8f0f+FLXbgjCc8Z8rPvb8+7vPwvOfBj38MBx44op9DD4V58+D00+GDHxzZ//nPw5OeBN/+Nnx8ZH0cfTRstBEcfzwcfvjI/q99DdZeG448srsMd8op8MhHwmGHwQknjOw/88zu58c+BiedtHjfKqvAd77TLR90EJxxxuL9a60FJ57YLb/znXD22Yv3b7ghHHNMt7zffnDBBYv3P/GJcET/mO69N/z854v3z5vXPX4Ar3sdXHvt4v3PfS58uD/mr3kN3Hzz4v1bbQXvfne3vM02cOedi/dvtx287W3d8vDfeTCp33uHDt23we1M4Pcer3xl9zt3n31G9r/rXbD11t3jtt9+I/sPPtjnHjT/3Jvqa+6D7bm32Dnmc69bfig+9wY48itJkqRmpKqW2s7mz59fCxcuXGr7GzL3gJOX+j5b8Lt+NPWx/eiyZtZVh7xiWZeg5cCCfgTlzKERJUkzynPsoSvJeVU1f3i7I7+SJElqhuFXkiRJzTD8SpIkqRmGX0mSJDXD8CtJkqRmGH4lSZLUDMOvJEmSmmH4lSRJUjMMv5IkSWqG4VeSJEnNMPxKkiSpGYZfSZIkNcPwK0mSpGasuKwLkCRJ45t7wMnLuoSHpN9deTPg4ztbrjrkFcu6hBHGDb9JNgL+A3gscB9wRFV9Ksn7gDcCi/pVD6yqU2arUEnLN184ZocvzLNreXxhljS7JjLyew+wf1Wdn2RV4Lwkp/V9n6yqj81eeZIkSdLMGTf8VtX1wPX98h1JLgM2mO3CJEmSpJk2qQ+8JZkLPBM4p2/aN8lFSb6UZI2ZLk6SJEmaSRMOv0keDZwI7FdVtwOHA48H5tGNDH98jNvtnWRhkoWLFi0abRVJkiRpqZhQ+E2yEl3w/UpVfR2gqm6oqnur6j7gC8AWo922qo6oqvlVNX/OnDkzVbckSZI0aeOG3yQBvghcVlWfGGhfb2C1HYBLZr48SZIkaeZM5NsetgT2AC5OckHfdiCwW5J5QAFXAfvMQn2SJEnSjJnItz38CMgoXX6nryRJkh5U/PfGkiRJaobhV5IkSc0w/EqSJKkZhl9JkiQ1w/ArSZKkZhh+JUmS1IyJfM+vNKrH7n7Isi5BkiRpUhz5lSRJUjMMv5IkSWqG4VeSJEnNMPxKkiSpGYZfSZIkNcPwK0mSpGYYfiVJktQMw68kSZKaYfiVJElSMwy/kiRJaobhV5IkSc0w/EqSJKkZhl9JkiQ1w/ArSZKkZqy4rAuQJI3tsbsfsqxLkB7SPMfa48ivJEmSmmH4lSRJUjMMv5IkSWqG4VeSJEnNMPxKkiSpGYZfSZIkNcPwK0mSpGYYfiVJktQMw68kSZKaYfiVJElSMwy/kiRJaobhV5IkSc0w/EqSJKkZhl9JkiQ1w/ArSZKkZhh+JUmS1AzDryRJkpph+JUkSVIzxg2/STZK8v0klyW5NMlb+/Y1k5yW5Bf9zzVmv1xJkiRp6iYy8nsPsH9VPQX4a+AfkmwGHACcUVWbAmf01yVJkqTl1rjht6qur6rz++U7gMuADYDtgaP61Y4CXj1LNUqSJEkzYlJzfpPMBZ4JnAOsW1XXQxeQgXVmvDpJkiRpBk04/CZ5NHAisF9V3T6J2+2dZGGShYsWLZpKjZIkSdKMmFD4TbISXfD9SlV9vW++Icl6ff96wI2j3baqjqiq+VU1f86cOTNRsyRJkjQlE/m2hwBfBC6rqk8MdH0L2LNf3hP45syXJ0mSJM2cFSewzpbAHsDFSS7o2w4EDgFOSPIG4DfATrNSoSRJkjRDxg2/VfUjIGN0bzWz5UiSJEmzx//wJkmSpGYYfiVJktQMw68kSZKaYfiVJElSMwy/kiRJaobhV5IkSc0w/EqSJKkZhl9JkiQ1w/ArSZKkZhh+JUmS1AzDryRJkpph+JUkSVIzDL+SJElqhuFXkiRJzTD8SpIkqRmGX0mSJDXD8CtJkqRmGH4lSZLUDMOvJEmSmmH4lSRJUjMMv5IkSWqG4VeSJEnNMPxKkiSpGYZfSZIkNcPwK0mSpGYYfiVJktQMw68kSZKaYfiVJElSMwy/kiRJaobhV5IkSc0w/EqSJKkZhl9JkiQ1w/ArSZKkZhh+JUmS1AzDryRJkpph+JUkSVIzDL+SJElqhuFXkiRJzTD8SpIkqRmGX0mSJDXD8CtJkqRmjBt+k3wpyY1JLhloe1+S3ya5oL9sO7tlSpIkSdM3kZHfI4GXj9L+yaqa119OmdmyJEmSpJk3bvitqh8AtyyFWiRJkqRZNZ05v/smuaifFrHGWCsl2TvJwiQLFy1aNI3dSZIkSdMz1fB7OPB4YB5wPfDxsVasqiOqan5VzZ8zZ84UdydJkiRN35TCb1XdUFX3VtV9wBeALWa2LEmSJGnmTSn8Jllv4OoOwCVjrStJkiQtL1Ycb4UkxwILgLWTXAu8F1iQZB5QwFXAPrNXoiRJkjQzxg2/VbXbKM1fnIVaJEmSpFnlf3iTJElSMwy/kiRJaobhV5IkSc0w/EqSJKkZhl9JkiQ1w/ArSZKkZhh+JUmS1AzDryRJkpph+JUkSVIzDL+SJElqhuFXkiRJzTD8SpIkqRmGX0mSJDXD8CtJkqRmGH4lSZLUDMOvJEmSmmH4lSRJUjMMv5IkSWqG4VeSJEnNMPxKkiSpGYZfSZIkNcPwK0mSpGYYfiVJktQMw68kSZKaYfiVJElSMwy/kiRJaobhV5IkSc0w/EqSJKkZhl9JkiQ1w/ArSZKkZhh+JUmS1AzDryRJkpph+JUkSVIzDL+SJElqhuFXkiRJzTD8SpIkqRmGX0mSJDXD8CtJkqRmGH4lSZLUDMOvJEmSmjFu+E3ypSQ3JrlkoG3NJKcl+UX/c43ZLVOSJEmavomM/B4JvHxY2wHAGVW1KXBGf12SJElaro0bfqvqB8Atw5q3B47ql48CXj2zZUmSJEkzb6pzftetqusB+p/rjLVikr2TLEyycNGiRVPcnSRJkjR9s/6Bt6o6oqrmV9X8OXPmzPbuJEmSpDFNNfzekGQ9gP7njTNXkiRJkjQ7php+vwXs2S/vCXxzZsqRJEmSZs9EvursWOBs4ElJrk3yBuAQ4CVJfgG8pL8uSZIkLddWHG+FqtptjK6tZrgWSZIkaVb5H94kSZLUDMOvJEmSmmH4lSRJUjMMv5IkSWqG4VeSJEnNMPxKkiSpGYZfSZIkNcPwK0mSpGYYfiVJktQMw68kSZKaYfiVJElSMwy/kiRJaobhV5IkSc0w/EqSJKkZhl9JkiQ1w/ArSZKkZhh+JUmS1AzDryRJkpph+JUkSVIzDL+SJElqhuFXkiRJzTD8SpIkqRmGX0mSJDXD8CtJkqRmGH4lSZLUDMOvJEmSmmH4lSRJUjMMv5IkSWqG4VeSJEnNMPxKkiSpGYZfSZIkNcPwK0mSpGYYfiVJktQMw68kSZKaYfiVJElSMwy/kiRJaobhV5IkSc0w/EqSJKkZhl9JkiQ1w/ArSZKkZqw4nRsnuQq4A7gXuKeq5s9EUZIkSdJsmFb47b24qm6age1IkiRJs8ppD5IkSWrGdMNvAacmOS/J3jNRkCRJkjRbpjvtYcuqui7JOsBpSS6vqh8MrtCH4r0BNt5442nuTpIkSZq6aY38VtV1/c8bgW8AW4yyzhFVNb+q5s+ZM2c6u5MkSZKmZcrhN8mjkqw6tAy8FLhkpgqTJEmSZtp0pj2sC3wjydB2vlpV352RqiRJkqRZMOXwW1VXAs+YwVokSZKkWeVXnUmSJKkZhl9JkiQ1w/ArSZKkZhh+JUmS1AzDryRJkpph+JUkSVIzDL+SJElqhuFXkiRJzTD8SpIkqRmGX0mSJDXD8CtJkqRmGH4lSZLUDMOvJEmSmmH4lSRJUjMMv5IkSWqG4VeSJEnNMPxKkiSpGYZfSZIkNcPwK0mSpGYYfiVJktQMw68kSZKaYfiVJElSMwy/kiRJaobhV5IkSc0w/EqSJKkZhl9JkiQ1w/ArSZKkZhh+JUmS1AzDryRJkpph+JUkSVIzDL+SJElqhuFXkiRJzTD8SpIkqRmGX0mSJDXD8CtJkqRmGH4lSZLUDMOvJEmSmmH4lSRJUjMMv5IkSWqG4VeSJEnNmFb4TfLyJFck+WWSA2aqKEmSJGk2TDn8JlkB+BywDbAZsFuSzWaqMEmSJGmmTWfkdwvgl1V1ZVX9BTgO2H5mypIkSZJm3nTC7wbANQPXr+3bJEmSpOXSitO4bUZpqxErJXsDe/dX/5DkimnsU8uftYGblnURD0X5yLKuQMsRz7NZ4nmmnufYLFnG59gmozVOJ/xeC2w0cH1D4LrhK1XVEcAR09iPlmNJFlbV/GVdh/RQ5nkmzS7PsbZMZ9rDucCmSf4qycOBXYFvzUxZkiRJ0syb8shvVd2TZF/ge8AKwJeq6tIZq0ySJEmaYdOZ9kBVnQKcMkO16MHJKS3S7PM8k2aX51hDUjXiM2qSJEnSQ5L/3liSJEnNMPxKkiSpGYZfjSvJE5OckuSXSS5LckKSdZMsSHLSsHWPTLJjv3xmkvn98lVJThxYb8ckR/bLeyVZlOSCgctmSR6W5NNJLklycZJzk/zVwPYuHlj/00vtAZFmWZK5SS7plxckqSSvHOg/KcmCfvnMJFckuSjJ5Uk+m+QxA+v+Ydi2/ynJXUlWH2hb4j6kh6L+PKskBw20rZ3k7iSfHbbuhUmOHdZ2/+vdKNt+Zr/tl/XXD04e+MbbJJskuXLwXNXSY/h9CEhnVo5lkpWBk4HDq+oJVfUU4HBgzhQ2Nz/JU8foO76q5g1cfgbsAqwPbF5VTwd2AG4buM2LB9b/xynUI03ZbJ53o7gW+Jcl9L+2qjYHNgf+DHxzCevuRvdVlTtMch/SUrcUzrMrge0Gru8ELPbNVUmeQpeXXpjkURPc7m7Aj/qfAAcB2/fbAvgU8O6qum2KdWsaDL8PUv1frJclOQw4H/hikoVJLk3y/oH1np3kx/1frT9JsmqSFZJ8tB9JvSjJPkvY1e7A2VX17aGGqvp+VV0yhbI/Bhw4ifXXA66vqvv6/V5bVbdOYb/SjFiK591wFwK/T/KSJa1UVX8B3g5snOQZo9T/eODRwLt44EV5UvuQZttSPs/uBC5L/y4l3aDLCcPW2R04GjgVeNUE6g+wI7AX8NIkK1fVncA/A4cl2QZYtaq+Mu6DoVkxra860zL3JOBvq+rNSdasqluSrACckWRz4HLgeGCXqjo3yWp0J/obgN9X1bOTPAI4K8mpVfXrUfbxNOC8JdTwgiQXDFzfGDhpjHVPAN6c5Amj9O2S5PkD15/br/+jJC8AzgCOqaqfDqzz/ST39stHVdUnl1CnNFOWxnk3mg/2l9OWtFJV3ZvkQuDJdIF20G7AscAPgSclWaeqbpzsPqSlYGmeZ8cBuyb5HXAv3X+rXX+gfxfgJX1N+9KdQ0uyJfDrqvpVkjOBbYGvV9UpSd4A/Afw/CVtQLPL8PvgdnVV/W+/vHOSvemO6XrAZkDRjZyeC1BVtwMkeSmw+cBcpdWBTYGJvggP+mFV3f+WUfp5vGO4F/go8E7gO8P6jq+qfYe1XZvkScDf9JczkuxUVWf0/S+uKv8Xu5a2ZXLeVdUPk9D/MTiejNG+K7BDVd2X5Ot0b/F+bor7kGbT0jzPvks3LeEGukB9vyTPBhZV1dVJrgW+lGSNcd6F3I0uUNP/3AP4en/9c8AqVXXFEu+9ZpXh98HtjwDpPgT2NuDZVXVrH0BXpnsBHO2LnAO8paq+N4F9XAq8aGbKBbq3jt7JsDlVY6mqP9MF5e8kuQF4Nd0osLSsLI3zbiwfopuXe89YK/SjY08HLhvWvjldCDite1eWh9PNd/zcsE2Muw9pKVhq51lV/SXJecD+wFOBVw507wY8OclV/fXVgNcA/z7atvrz7zXAq5L8S1/PWklWrao7gPv6i5Yh5/w+NKxG94vi90nWBbbp2y8H1u//cqWfD7Ui3b+kflOSlfr2J2bsSfxfBZ6X5BVDDUlenuTpUym0qu4GPgnsN966SZ6VZP1++WF0H+a5eir7lWbBbJ53o6qqU4E1gBHzefttrgR8GLimqi4a1r0b8L6qmttf1gc2SLLJZPYhLWVL6zz7OPCOqrp5qKF/3dmJ7kPXc6tqLrA9I+fLD9oauLCqNupvswlwIt3AjZYTjvw+BFTVhUl+SjeaeiVwVt/+lyS7AJ9JsgrdfKit6f5inQuc30/MX8QYJ2ZV3ZlkO+DQJIcCdwMXAW8F1ppiyV+k+8DNoOFzft9M90vvC/28LYCfAINfPzM45/eiqnr9FOuRJm02z7txfIiR3+bwlSR/Bh4BnE73Aj3crjwQHIZ8o28/ZwL7kJa6pXWeVdWljHxH8oXAb6vqtwNtPwA2S7Jef/3z/WsjwDXAFXTn1aATgTfRvfOp5YD/3liSJEnNcNqDJEmSmuG0BwHQz+Ed/pbMn6vqOcuiHqkFnnfS7PM803BOe5AkSVIznPYgSZKkZhh+JUmS1AzDryRJkpph+JUkSVIzDL+SJElqxv8HBJeBc1T/ZxYAAAAASUVORK5CYII=\n", + "text/plain": [ + "
    " + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "# ROC/Min Max Threshold Test\n", + "\n", + "threshold_test_gender = MinMaxMetricThreshold(\n", + " #test_name=\"\", # Default test name and description will be used accordingly if not specified\n", + " #test_desc=\"\",\n", + " attr=\"sex\",\n", + " metric=\"mae\",\n", + " threshold=27,\n", + ")\n", + "threshold_test_gender.run(df_test_with_output=output)\n", + "threshold_test_gender.plot(alpha=0.05)\n", + "\n", + "threshold_test_race = MinMaxMetricThreshold(\n", + " attr=\"race\",\n", + " metric=\"mae\",\n", + " threshold=27,\n", + ")\n", + "threshold_test_race.run(df_test_with_output=output)\n", + "threshold_test_race.plot(alpha=0.05)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsYAAAF1CAYAAADr3izzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZOElEQVR4nO3de7xvd13f+deHhDsiIAEhiQYkRcFa7SOi1I5zFFpQtPBox0eDl4ZKh6K0VYdODYzWKzPUcVrb6eCYein1QoyMSsZaFdNib4oG5aEDyBC5hgQISkQZBcHv/PFbx/44OZcdz945Sfp8Ph6/x16X71rr81tr7XPe+7u/67dnrRUAAPzX7h7nugAAALgzEIwBACDBGAAAKsEYAAAqwRgAACrBGAAAKsEY4KPMzL+cmW8/13VwdGbmkplZM3P+ua4FuHMRjOFuYGbeOjMfmpmHnrD8tVsAuOQc1PTCmXnLzPz+zNw4Mz96R9dw2GbmWTPzke097b8eea5rA+DsCcZw9/GW6pnHZ2bmz1b3PReFzMwV1VdUT15rPaC6rLruHNRxFD2Cv7jWesAJr5sOcuzbW89R9mjqLT0cziPcvQjGcPfxg9Xf2Ju/ovpX+w1m5t4z850z8/aZeffM/J8zc99t3YNn5qdm5paZed82fdHetq+amW+bmf80M783Mz93Yg/1ns+sfnat9VtVa613rbWu2tvXo2bmF7b9vHJm/vnM/NC27tjM3HhC3W+dmSdv00+YmV+cmVtn5uZt23vttV0z87yZeVP1pm3ZF22957fOzH+emU/ba/8ZM/OrWy0/Wt3nwGf8BFudXz8zv159YGYes9Xz7Jl5e/VvZ+YeM/MNM/O2mXnPzPyrmfnYbftLTmx/kmOc6To9ZGZ+YGZu2tb/5P553ep7V/UD2/3wXVvbm7bpe2/tH7rt+9aZ+Z2Z+Q8zc49t3dfPzDu3c/bGmXnSKc7HF87M67d275yZv7+37qTXZGb++sy8eWYeuM1/wcy8a2YuOMn+7zMzPzQzv73t51dm5uF71+LJe22/+fg9tucrt/d988w8f6/tfWfmpdv5e8PM/IP9e/Ik1/n8mfkrM/O6rY5Xzcyn7LVfM/OYvfk/Ga6zd11eODPv3fb9ZQc5h8DhE4zh7uOXqgfOzKfMzHnVX69ODAL/qPoz1adXj6kurP7htu4e1Q9Un1h9QvUH1T8/Yfsvrf5m9bDqXtWp/pP+pepvzMz/ODOXbfXs+5HqNdVDq29rF+IP6iPV123bPrF6UvXVJ7R5RvVZ1eNm5s9X31/97erjqu+prt1C4b2qn2z3Q8VDqh+r/trtqOVknlk9rXpQ9eFt2X9bfUr1lOpZ2+vzqkdXD+i253m//YnOdJ1+sLpf9fh21+mf7K37+Hbv8xOr51T/U/XZ7e6HP1c9ofqGre3zqxurC6qHVy+s1sw8tvo71WeutT5mq/GtpzgX31f97a3dp7YF/dNdk7XWj1a/WP2zmfm4bR9/a611y0n2f0X1sdXF236eu52Pg/q86tLqL1dX7gXpb6ouaXd9/lL15SfZdv86P7p6WfW17c7XT1f/9+z9wHYGH9/ufr5we09Xbee5TnEOgSOy1vLy8rqLv9oFkye3CzX/S/XU6pXV+dVq95/8VB+oPmlvuydWbznFPj+9et/e/Kuqb9ib/+rqZ05T05dVP78d87erK7fln9AuMN5/r+2PVD+0TR+rbjzZ+zvFcb62+om9+VV9/t78d1ffdsI2b2wXPj+3uqmavXX/ufr2UxzrWVvtt+69fuuEOr9yb/6SrZ5H7y27rvrqvfnHVn+0XavbtD/Atf+T61Q9ovrj6sEnaXes+lB1n71lv1V94d78U6q3btPfWr2ieswJ+3lM9Z7tfrvnGWp7e7vw+8ATlp/ymmzTD9q2/Y3qe06z/6/crtennep7Ym/+m/fusePn+ZP31n9H9X3b9Jurp+yt+1v79+RJrvM3Vtfszd+jemd1bO+efMze+n95/B7brsuJ3w/XVN94unPo5eV1NC89xnD38oPtenWf1QnDKNr1ZN2ves32695bq5/Zljcz95uZ79l+xf/+6t9XDzqht/dde9P/X7vezpNaa/3wWuvJ7ULOc6tvnZmnVI9sF+Q+sNf8bQd9gzPzZ7Zf8b9rq/N/btfbtu8de9OfWD3/+Hve3vfFWx2PrN651lq3o5ZfWms9aO/1Sac59smWPfKEY7ytXSh++Bn2UZ3xOl1c/c5a632n2PyWtdYfnqGW4w8S/q/VDdXPbUMbrqxaa93Q7oeRb67eMzNXz6kfPvxr1RdWb5vd0JknbstPd01aa93arvf+U6v/7VTnot39/rPV1duQiO+YmXuepv2J9s/z/nt/5Anrbtc1XWv98bb+wgPWcbLvh+O1nOocAkdAMIa7kbXW29o9hPeF1Y+fsPq97X7N/Pi9UPexa/dwXO1+df7Y6rPWWg9s15tau57ms6npj9ZaP1b9erugc3P14Jm5/16zT9ib/kC7AL87+C7w7Y8v/e7qN6tLtzpfeJIa94PuO6oXnRBm77fWetlWy4Uzs7/9J3R21hmW3dQuGO4f78PVu8+wj+NOd53eUT1kZh50wNpOVstNVWut31trPX+t9ejqi6v/4fhY4rXWj6y1/uK27Wo3ROe2B1vrV9ZaT283pOMn2/WE1umvSTPz6e16g19W/bNTnYjt3vqWtdbjqr9QfVH/ZZz9R91H7YYrnOjik733dvfFRado9yeH35v+qPO43U8Xt+s1rt0Pkaer5WTfD8evw6nOIXAEBGO4+3l2u6EE+z1Qx3ux/kX1T2bmYVUzc+HWi1v1Me2C860z85B24yz/VGb3sWZPm5mPmd3DZl/Qbszrq7fwfn31LTNzr5n5i+2C13H/b3Wfbft7thsecu+99R9Tvb/6/Zn55OqrzlDOv6ieOzOfNTv3P15bu7GsH67+3vYA1V9tN872KL2s+rrZPYD4gHY93j+61vrwGbY77pTXaa11c/VvqpfM7iG9e87M555iP8dr+YaZuWB2D1L+w7Zx6bN7OO4xW8h7f7ux3R+ZmcfOzOfP7iG9P9xq+ciJO96u7ZfNzMeutf5obx91mmsyM/fZanhhu/HsF87MiWPIjx/j82bmz24/PL2/3ZCU48d4bXX5dg4uq/67k+ziG7ce+Mdvxzr+kYLXVC/YzuGF7cZUn8411dNm5knbPfv86oPthnkcr+VLZ+a8mXlqu2E8Jzr+/fDftAv4P3aGcwgcAcEY7mbWWr+11rr+FKu/vt2vx39p+zX8z7frfaz6rnYf7/bedg/P/cxZlPH+dsHm7e3G4X5H9VVrrf+4rf/Sdg/H/U67YPcnwz7WWr/bbvzy97brcftAu4fAjvv72/a/1y5gnfbzkbdz8d+3e0Dtfe3e/7O2dR+q/uo2/752Dyye2NN+oifObT/H+DPPsM2+7283BODft+vd/8Pq796O7b+r01+nr2gXEH+z3Vjgrz3Nvr693Q8pv95uPO+vbstq91Daz1e/3+4HiJestV7V7oeUF2/Hf1e7nswXnmL/X1G9dbvXntv2ENvprkm7MfI3rrW+e631wW2bb5+ZS0+y/4+vXt7ufntD9Qv9lwdOv7H6pG3/39JuHPuJfmE79nXVd661fm5b/q3t7rm3bOfg5e2C7kmttd641fm/tzsvX1x98XZ/VX3NtuzWdmPvf/KEXbxrq/Om6oer5661fnNbd9JzCByN+eihdQB3vJn55nYPJ/lPnzudmfmq6vK11sl6es9238faPRR40RmaAncAPcYAsGdmHjEzn7MNA3psu6ERP3Gu6wKOnr/YAwAf7V7tPlv5Ue2GP1xdveRcFgTcMQylAACADKUAAIBKMAYAgOpOMsb4oQ996LrkkkvOdRkAANzNveY1r3nvWuuCk627UwTjSy65pOuvP9XHrgIAwOGYmbedap2hFAAAkGAMAACVYAwAAJVgDAAAlWAMAACVYAwAAJVgDAAAlWAMAACVYAwAAJVgDAAAlWAMAACVYAwAAJVgDAAAlWAMJ3Xs2LGOHTt2rssAAO5AgjEAACQYAwBAJRgDAEAlGAMAQCUYAwBAJRgDAEAlGAMAQCUYAwBAJRgDAEAlGAMAQCUYAwBAJRgDAEAlGAMAQCUYAwBAJRgDAEfg2LFjHTt27FyXAbeLYAwAAAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUB0wGM/M183M62bm/5mZl83MfWbmITPzypl50/b1wXvtXzAzN8zMG2fmKUdXPgAAHI4zBuOZubD6e9Vla61Prc6rLq+urK5ba11aXbfNNzOP29Y/vnpq9ZKZOe9oygcAgMNx0KEU51f3nZnzq/tVN1VPr166rX9p9Yxt+unV1WutD6613lLdUD3h0CoGAIAjcMZgvNZ6Z/Wd1durm6vfXWv9XPXwtdbNW5ubq4dtm1xYvWNvFzduyz7KzDxnZq6fmetvueWWs3sXAABwlg4ylOLB7XqBH1U9srr/zHz56TY5ybJ1mwVrXbXWumytddkFF1xw0HoBAOBIHGQoxZOrt6y1bllr/VH149VfqN49M4+o2r6+Z2t/Y3Xx3vYXtRt6AQAAd1oHCcZvrz57Zu43M1M9qXpDdW11xdbmiuoV2/S11eUzc++ZeVR1afXLh1s2AAAcrvPP1GCt9eqZeXn1q9WHq1+rrqoeUF0zM89uF56/ZGv/upm5pnr91v55a62PHFH9AABwKM4YjKvWWt9UfdMJiz/Yrvf4ZO1fVL3o7EoDAIA7jr98BwAACcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUNX557qAc+2SK//1uS6BO6F3vfm3K/cHJ/fWFz/tXJcAwBHQYwwAAAnGAABQCcYAAFAJxgAAUHn4DoCz5CFVTsZDzJzOnfUhZj3GAACQYAwAAJVgDAAAlWAMAACVYAwAAJVgDAAAlWAMAACVYAwAAJVgDAAAlWAMAACVYAwAAJVgDAAAlWAMAACVYAwAAJVgDAAAlWAMAACVYAwAAJVgDAAAlWAMAACVYAwAAJVgDAAAlWAMAACVYAwAAJVgDAAAlWAMAACVYAwAAJVgDAAAlWAMAACVYAwAAJVgDAAAlWAMAADVAYPxzDxoZl4+M785M2+YmSfOzENm5pUz86bt64P32r9gZm6YmTfOzFOOrnwAADgcB+0x/qfVz6y1Prn6c9Ubqiur69Zal1bXbfPNzOOqy6vHV0+tXjIz5x124QAAcJjOGIxn5oHV51bfV7XW+tBa69bq6dVLt2YvrZ6xTT+9unqt9cG11luqG6onHG7ZAABwuA7SY/zo6pbqB2bm12bme2fm/tXD11o3V21fH7a1v7B6x972N27LPsrMPGdmrp+Z62+55ZazehMAAHC2DhKMz6/+fPXda63PqD7QNmziFOYky9ZtFqx11VrrsrXWZRdccMGBigUAgKNykGB8Y3XjWuvV2/zL2wXld8/MI6q2r+/Za3/x3vYXVTcdTrkAAHA0zhiM11rvqt4xM4/dFj2pen11bXXFtuyK6hXb9LXV5TNz75l5VHVp9cuHWjUAAByy8w/Y7u9WPzwz96reXP3NdqH6mpl5dvX26kuq1lqvm5lr2oXnD1fPW2t95NArBwCAQ3SgYLzWem112UlWPekU7V9UvehPXxYAANyx/OU7AABIMAYAgOrgY4zhvyof/6UvPtclAAB3MMEYADh0Ohi4KzKUAgAAEowBAKASjAEAoBKMAQCgEowBAKASjAEAoBKMAQCgEowBAKASjAEAoBKMAQCgEowBAKASjAEAoBKMAQCgEowBAKASjAEAoBKMAQCgEowBAKASjAEAoBKMAQCgEowBAKASjAEAoBKMAQCgEowBAKASjAEAoBKMAQCgEowBAKASjAEAoBKMAQCgEowBAKASjAEAoBKMAQCgEowBAKASjAEAoBKMAQCgEowBAKASjAEAoBKMAQCgEowBAKASjAEAoBKMAQCgEowBAKASjAEAoBKMAQCgEowBAKASjAEAoBKMAQCgEowBAKASjAEAoBKMAQCgEowBAKASjAEAoBKMAQCgEowBAKC6HcF4Zs6bmV+bmZ/a5h8yM6+cmTdtXx+81/YFM3PDzLxxZp5yFIUDAMBhuj09xl9TvWFv/srqurXWpdV123wz87jq8urx1VOrl8zMeYdTLgAAHI0DBeOZuah6WvW9e4ufXr10m35p9Yy95VevtT641npLdUP1hEOpFgAAjshBe4y/q/oH1R/vLXv4Wuvmqu3rw7blF1bv2Gt347YMAADutM4YjGfmi6r3rLVec8B9zkmWrZPs9zkzc/3MXH/LLbcccNcAAHA0DtJj/DnVX5mZt1ZXV58/Mz9UvXtmHlG1fX3P1v7G6uK97S+qbjpxp2utq9Zal621LrvgggvO4i0AAMDZO2MwXmu9YK110VrrknYP1f3btdaXV9dWV2zNrqhesU1fW10+M/eemUdVl1a/fOiVAwDAITr/LLZ9cXXNzDy7env1JVVrrdfNzDXV66sPV89ba33krCsFAIAjdLuC8VrrVdWrtunfrp50inYvql50lrUBAMAdxl++AwCABGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoBGMAAKgEYwAAqARjAACoDhCMZ+bimfl3M/OGmXndzHzNtvwhM/PKmXnT9vXBe9u8YGZumJk3zsxTjvINAADAYThIj/GHq+evtT6l+uzqeTPzuOrK6rq11qXVddt827rLq8dXT61eMjPnHUXxAABwWM4YjNdaN6+1fnWb/r3qDdWF1dOrl27NXlo9Y5t+enX1WuuDa623VDdUTzjkugEA4FDdrjHGM3NJ9RnVq6uHr7Vurl14rh62NbuwesfeZjduy07c13Nm5vqZuf6WW275U5QOAACH58DBeGYeUP1f1deutd5/uqYnWbZus2Ctq9Zal621LrvgggsOWgYAAByJAwXjmblnu1D8w2utH98Wv3tmHrGtf0T1nm35jdXFe5tfVN10OOUCAMDROMinUkz1fdUb1lr/eG/VtdUV2/QV1Sv2ll8+M/eemUdVl1a/fHglAwDA4Tv/AG0+p/qK6jdm5rXbshdWL66umZlnV2+vvqRqrfW6mbmmen27T7R43lrrI4ddOAAAHKYzBuO11n/s5OOGq550im1eVL3oLOoCAIA7lL98BwAACcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUAnGAABQCcYAAFAJxgAAUB1hMJ6Zp87MG2fmhpm58qiOAwAAh+FIgvHMnFf9H9UXVI+rnjkzjzuKYwEAwGE4qh7jJ1Q3rLXevNb6UHV19fQjOhYAAJy1owrGF1bv2Ju/cVsGAAB3Sucf0X7nJMvWRzWYeU71nG3292fmjUdUC/xpPbR677kugjuf+UfnugK4y/DvKCd1jv8d/cRTrTiqYHxjdfHe/EXVTfsN1lpXVVcd0fHhrM3M9Wuty851HQB3Vf4d5a7mqIZS/Ep16cw8ambuVV1eXXtExwIAgLN2JD3Ga60Pz8zfqX62Oq/6/rXW647iWAAAcBiOaihFa62frn76qPYPdwBDfQDOjn9HuUuZtdaZWwEAwN2cPwkNAAAJxnAbM/ORmXnt3uuSc10TwF3FzKyZ+cG9+fNn5paZ+alzWRccxJGNMYa7sD9Ya336uS4C4C7qA9Wnzsx911p/UP2l6p3nuCY4ED3GAMBh+zfV07bpZ1YvO4e1wIEJxnBb990bRvET57oYgLugq6vLZ+Y+1adVrz7H9cCBGEoBt2UoBcBZWGv9+vZ8xjPz0a3chQjGAMBRuLb6zupY9XHnthQ4GMEYADgK31/97lrrN2bm2DmuBQ5EMAYADt1a68bqn57rOuD28JfvAAAgn0oBAACVYAwAAJVgDAAAlWAMAACVYAwAAJVgDAAAlWAMAACVYAwAAFX9/862TVD3XhCzAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
    " + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAswAAAF1CAYAAAD8/Lw6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAeiklEQVR4nO3df7xlZV0v8M9XRkFEA2RAnBkdFMLAmz8aUW/eGoNeUHQvZHIF06BISim1SyaYr+SqFN16Xe1WaGQq+AvITLllpqJc86Zwxx9piAg4CCO/xh+IUKLgc/9Ya3Rz5pznzJwz55zx8H6/Xvu1137Ws9bz7L3WnvmcZz9r72qtBQAAmN79lroDAACwMxOYAQCgQ2AGAIAOgRkAADoEZgAA6BCYAQCgQ2AG6KiqN1fVq5e6Hyy+qjqpqj661P0Alp7ADMtQVV1XVd+uqn2mlH+6qlpVrV2CPr2sqjZW1R1VtamqLlzsPuxoY6C6Z3xOk7eHL3XfANhxBGZYvjYmOWHLg6r6D0keuBQdqaoTkzw3yRGttT2SrEtyyRL0Y8UC7PZjrbU9ptxu3Ja2t7c/C9T/Bd93p81dFrvNpbIUry+w4wjMsHy9JckvTTw+Mcn5kxWqateq+uOqur6qbqmq11fVA8d1e1XV31XV5qr6+ri8emLbS6vqVVX1f6vqm1X1/qkj2hOelOQfW2vXJklr7ebW2rkT+zqgqv7PuJ8PVNWfVdVbx3Xrq2rTlH5fV1VHjMuHVdXHquq2qrpp3PYBE3VbVZ1aVVcnuXos+7lxtP22qvrnqvrRifpPqKpPjn25MMlu2/yKTzH286VV9Zkkd1bVgWN/Tq6q65N8qKruV1Uvr6ovVdWtVXV+Vf3QuP3aqfWnaWO247R3Vb2pqm4c17978nUd+3dzkjeN58Nrx7o3jsu7jvX3Gfd9W1V9rar+qaruN657aVV9eXzNrqqqw2d4Pd5cVa+rqvdW1Z1Jnl5VR1fVp6rq9qq6oarOnLLN08ZjdNu4/qSxfMZzd5p2DxzPr29U1VfG4zr5+q6YqHtpVf3qvTevPx23/fzkcxvP24+Mz/uDVfXnE+ftVsdulmM923l+ZlW9s6ouHNv7ZFU9bqLuNh0DYG4EZli+Pp7kIVX1IzWM5D0ryVun1PnDJD+c5PFJDkyyKsnvjevul+RNSR6Z5BFJ/j3Jn03Z/tlJfjnJvkkekOS3O335pap6SVWtq61HFt+e5BNJ9knyqgzhflvdk+S3xm2fmuTwJC+YUufYJE9OckhVPTHJG5P8WpKHJvmLJBePAewBSd6d4Y+NvZP8dZJf2I6+TOeEJEcn2TPJ3WPZTyb5kSRHJjlpvD09yaOS7JGtX+fJ+lPNdpzekmT3JIdmOE6vmVj3sAzP85FJTknyu0mekuF8eFySw5K8fKx7WpJNSVYm2S/Jy5K0qjo4yW8keVJr7cFjH6/rvB7PTnJWkgcn+WiSOzP8Ybdnhtfp+VV1bJJU1SOS/EOSPx3bfXyST4/76Z27U70qyfuT7JVk9bi/bfXkJF/McH69Ism7qmrvcd3bk1ye4Tw6M8OnKFNt77HuOSbDObn32Pa7q+r+czgGwPZqrbm5uS2zW4b/LI/IEHb+IMlRST6QZEWSlmRtksoQVh49sd1Tk2ycYZ+PT/L1iceXJnn5xOMXJHlfp0+/mOSDY5tfTXL6WP6IDEHyQRN1357krePy+iSbpnt+M7Tz4iR/O/G4JfmpicevS/KqKdtclSHY/ESSG5PUxLp/TvLqGdo6aez7bRO3a6f081cmHq8d+/OoibJLkrxg4vHBSb4zHqut6m/Dsf/ecUqyf5LvJtlrmnrrk3w7yW4TZdcm+dmJx0cmuW5cfmWS9yQ5cMp+Dkxy63i+3X+Wvr05yfmz1HltkteMy2dMHsuJOtt77p6f5Nwkq6eUb3l9V0w5r3914vhOPR8uzxCMt5y3u0+se2u+f95u77Fen855niGQf3xi3f2S3JTkP23PMXBzc5vbzQgzLG9vyTCid1KmTMfIMGK3e5JPjB9335bkfWN5qmr3qvqL8ePj25N8JMmeU0aHb55Y/rcMI2bTaq29rbV2RIaRxF9P8sqqOjLJwzMEvDsnqn9pW59gVf3wOFXg5rGfv59hNHDSDRPLj0xy2pbnPD7vNWM/Hp7ky621th19+Xhrbc+J26M7bU9X9vApbXwpQ4Dab5Z9JJn1OK1J8rXW2tdn2Hxza+1bs/RlywWMf5TkmiTvr6ovVtXpSdJauybDHylnJrm1qi6o/kWP93ouVfXkqvpwDVNKvpHh3Nhy/NZkCPFTdc/dafxOhpB9eVVdUVW/0unfVNOdD1vOla+11v5tpuc2Tdm2HOue7+2rtfbdDCP+D5/DMQC2k8AMy1hr7UsZLv772STvmrL6Kxk+vj90Iuz9UBsuykuGj+APTvLk1tpDMoy+JkPwmE+fvtNa++skn0ny2AyjZHtV1YMmqj1iYvnODOFoaHwIgpPB6HVJPp/koLGfL5umj5OB54YkZ00Jubu31t4x9mVVVU1u/4jMT5ul7MYMIX6yvbuT3DLLPrboHacbkuxdVXtuY9+m68uNSdJa+2Zr7bTW2qOS/Ock/23LPNnW2ttba08bt20ZpkvMZGqbb09ycZI1rbUfSvL6fP/43ZBk6h8gyezn7r0bHObMP6+19vAMU3HOqaoDM5xbycT5lWGayqTpzocbM5wre1fV5LZrZnm+vWM923l+r/3XMH98db5/fLbnGADbSWCG5e/kDFMSJkdwt4xQ/WWS11TVvklSVavGUd9kmGP670luG+dsvmKuHajh69eOrqoHjxc+/UyGObWXjaF+Q5L/XlUPqKqnZQhkW3whyW7j9vfPMM1k14n1D05ye5I7quoxSZ4/S3f+MsmvjyObVVUP2tK3JB/LEGBeWFUrquoZGebxLqR3JPmt8QKyPTKMkF/YWrt7lu22mPE4tdZuyjAH+JwaLg68f1X9xAz72dKXl1fVyhou4Py9jPPea7hQ8sAxPN6eYe74PVV1cFX9VA0XB35r7Ms92/H8H5xhpPZbVXVYhk9EtnhbkiOq6r+Ox+OhVfX4bTh376WqjqvvXwj59QyB8p7W2uYkX07ynKraZRx5nhrQ981wPty/qo7LMB/5vRPn7ZnjefvU3Pu8nU7vWM92nifJj1XVM2q4SPHFSe5K8vEdcAyAWQjMsMy11q5trW2YYfVLM3zM/vHx4/wPZhitTIa5pA/MMJr38Qwfec/V7RlGfq/PMM/3fyR5fmtty49CPDvDxVVfyxD4vjd9pLX2jQzzo9+QIdzcmeGj6C1+e9z+mxlCVPf7ncfX4nkZLrb6eobnf9K47ttJnjE+/nqGCyWnjsxP9dTa+nuYnzTLNpPemGHqzEcyfBrwrSS/uR3bvzb94/TcDPNkP59hnuuLO/t6dYYQ+Jkkn03yybEsSQ7KcH7ckeEPi3Naa5dmCHVnj+3fnCFgvmw7+v+CDNNzvpkhoF+0ZUVr7foMn46cluHc+HSGixGT/rk71ZOSXFZVd2QYzX5Ra23juO55SV6SYV79oRnmrE+6bHzuX8lwseIzW2tfHdf9Yoa501/N8DpdmCHEzmTGY70N53kyzCF/VoZz87lJntFa+07mfwyAWdS9p2YBLL0avlrswNbac5a6L7Ctavi6us+31ub8aUxn32fGewKWjBFmAJiDqnpSVT16nGZ0VIavfXv3EncLWAB+eQgA5uZhGabsPDTD9Innt9Y+tbRdAhaCKRkAANBhSgYAAHQIzAAA0LHTz2HeZ5992tq1a5e6GwAALGOf+MQnvtJam/YXQ3f6wLx27dps2DDTV8gCAMD8VdWXZlpnSgYAAHQIzAAA0CEwAwBAh8AMAAAdAjMAAHQIzAAA0CEwAwBAh8AMAAAdAjMAAHQIzAAA0CEwAwBAh8AMAAAdAjMAAHQIzCyq9evXZ/369UvdDQCAbSYwAwBAh8AMAAAdAjMAAHQIzAAA0CEwAwBAh8AMAAAdAjMAAHQIzAAA0CEwAwBAx6yBuareWFW3VtW/TpTtXVUfqKqrx/u9JtadUVXXVNVVVXXkRPmPVdVnx3X/q6pqxz8dAADYsbZlhPnNSY6aUnZ6kktaawcluWR8nKo6JMnxSQ4dtzmnqnYZt3ldklOSHDTepu4TAAB2OrMG5tbaR5J8bUrxMUnOG5fPS3LsRPkFrbW7Wmsbk1yT5LCq2j/JQ1prH2uttSTnT2wDAAA7rbnOYd6vtXZTkoz3+47lq5LcMFFv01i2alyeWg4AADu1HX3R33TzklunfPqdVJ1SVRuqasPmzZt3WOcAAGB7zTUw3zJOs8h4f+tYvinJmol6q5PcOJavnqZ8Wq21c1tr61pr61auXDnHLgIAwPzNNTBfnOTEcfnEJO+ZKD++qnatqgMyXNx3+Tht45tV9ZTx2zF+aWIbAADYaa2YrUJVvSPJ+iT7VNWmJK9IcnaSi6rq5CTXJzkuSVprV1TVRUk+l+TuJKe21u4Zd/X8DN+48cAk/zDeAABgpzZrYG6tnTDDqsNnqH9WkrOmKd+Q5LHb1TsAAFhifukPAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAGWkfXr12f9+vVL3Q2AZUVgBgDYDv4wve8RmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA65hWYq+q3quqKqvrXqnpHVe1WVXtX1Qeq6urxfq+J+mdU1TVVdVVVHTn/7gMAwMKac2CuqlVJXphkXWvtsUl2SXJ8ktOTXNJaOyjJJePjVNUh4/pDkxyV5Jyq2mV+3QcAgIU13ykZK5I8sKpWJNk9yY1Jjkly3rj+vCTHjsvHJLmgtXZXa21jkmuSHDbP9gEAYEHNOTC31r6c5I+TXJ/kpiTfaK29P8l+rbWbxjo3Jdl33GRVkhsmdrFpLNtKVZ1SVRuqasPmzZvn2kUAAJi3+UzJ2CvDqPEBSR6e5EFV9ZzeJtOUtekqttbOba2ta62tW7ly5Vy7CAAA8zafKRlHJNnYWtvcWvtOkncl+Y9Jbqmq/ZNkvL91rL8pyZqJ7VdnmMIBAAA7rfkE5uuTPKWqdq+qSnJ4kiuTXJzkxLHOiUneMy5fnOT4qtq1qg5IclCSy+fRPgAALLgVc92wtXZZVb0zySeT3J3kU0nOTbJHkouq6uQMofq4sf4VVXVRks+N9U9trd0zz/4DAMCCmnNgTpLW2iuSvGJK8V0ZRpunq39WkrPm0yYAACwmv/QHAAAdAjMAAHTMa0rGcrb29L9f6i4sSzd/8atJvL4L5bqzj17qLgDAsmOEGQAAOgRmAADoEJgBAKBDYAYAgA6BGQAAOgRmAADoEJgBAKBDYAYAgA6BGQAAOgRmAADoEJgBAKBDYAYAgA6BGQAAOgRmAADoEJgBAKBDYAYAgA6BGQAAOgRmAADoEJgBAKBDYAYAgA6BGQAAOgRmAADoEJgBAKBDYAYAgA6BGQAAOgRmAADoWLHUHQDuu9ae/vdL3YVl5+YvfjWJ13ahXHf20UvdBWAJCMwAsEz5w2lh+MN0Ye2Mf5iakgEAAB0CMwAAdAjMAADQITADAECHwAwAAB0CMwAAdAjMAADQITADAECHwAwAAB0CMwAAdAjMAADQITADAECHwAwAAB0CMwAAdAjMAADQITADAECHwAwAAB0CMwAAdAjMAADQMa/AXFV7VtU7q+rzVXVlVT21qvauqg9U1dXj/V4T9c+oqmuq6qqqOnL+3QcAgIU13xHmP0nyvtbaY5I8LsmVSU5Pcklr7aAkl4yPU1WHJDk+yaFJjkpyTlXtMs/2AQBgQc05MFfVQ5L8RJK/SpLW2rdba7clOSbJeWO185IcOy4fk+SC1tpdrbWNSa5Jcthc2wcAgMUwnxHmRyXZnORNVfWpqnpDVT0oyX6ttZuSZLzfd6y/KskNE9tvGssAAGCnNZ/AvCLJE5O8rrX2hCR3Zpx+MYOapqxNW7HqlKraUFUbNm/ePI8uAgDA/KyYx7abkmxqrV02Pn5nhsB8S1Xt31q7qar2T3LrRP01E9uvTnLjdDturZ2b5NwkWbdu3bShmh9MD3v22UvdBQCA7TLnEebW2s1Jbqiqg8eiw5N8LsnFSU4cy05M8p5x+eIkx1fVrlV1QJKDklw+1/YBAGAxzGeEOUl+M8nbquoBSb6Y5JczhPCLqurkJNcnOS5JWmtXVNVFGUL13UlOba3dM8/2AQBgQc0rMLfWPp1k3TSrDp+h/llJzppPmwAAsJj80h8AAHQIzAAA0CEwAwBAh8AMAAAdAjMAAHQIzAAA0CEwAwBAh8AMAAAdAjMAAHQIzAAA0CEwAwBAh8AMAAAdAjMAAHQIzAAA0CEwAwBAh8AMAAAdAjMAAHSsWOoOALDjPOzZZy91F2DZ8z677zHCDAAAHQIzAAB0CMwAANAhMAMAQIfADAAAHQIzAAB0CMwAANAhMAMAQIfADAAAHQIzAAB0CMwAANAhMAMAQIfADAAAHQIzAAB0CMwAANAhMAMAQIfADAAAHQIzAAB0CMwAANAhMAMAQIfADAAAHQIzAAB0CMwAANAhMAMAQIfADAAAHQIzAAB0CMwAANAhMAMAQIfADAAAHQIzAAB0CMwAANAhMAMAQIfADAAAHfMOzFW1S1V9qqr+bny8d1V9oKquHu/3mqh7RlVdU1VXVdWR820bAAAW2o4YYX5RkisnHp+e5JLW2kFJLhkfp6oOSXJ8kkOTHJXknKraZQe0DwAAC2ZegbmqVic5OskbJoqPSXLeuHxekmMnyi9ord3VWtuY5Jokh82nfQAAWGjzHWF+bZLfSfLdibL9Wms3Jcl4v+9YvirJDRP1No1lW6mqU6pqQ1Vt2Lx58zy7CAAAczfnwFxVP5fk1tbaJ7Z1k2nK2nQVW2vnttbWtdbWrVy5cq5dBACAeVsxj21/PMl/qaqfTbJbkodU1VuT3FJV+7fWbqqq/ZPcOtbflGTNxPark9w4j/YBAGDBzXmEubV2RmttdWttbYaL+T7UWntOkouTnDhWOzHJe8bli5McX1W7VtUBSQ5Kcvmcew4AAItgPiPMMzk7yUVVdXKS65MclySttSuq6qIkn0tyd5JTW2v3LED7AACww+yQwNxauzTJpePyV5McPkO9s5KctSPaBACAxeCX/gAAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCAjjkH5qpaU1Ufrqorq+qKqnrRWL53VX2gqq4e7/ea2OaMqrqmqq6qqiN3xBMAAICFNJ8R5ruTnNZa+5EkT0lyalUdkuT0JJe01g5Kcsn4OOO645McmuSoJOdU1S7z6TwAACy0OQfm1tpNrbVPjsvfTHJlklVJjkly3ljtvCTHjsvHJLmgtXZXa21jkmuSHDbX9gEAYDHskDnMVbU2yROSXJZkv9baTckQqpPsO1ZbleSGic02jWXT7e+UqtpQVRs2b968I7oIAABzMu/AXFV7JPmbJC9urd3eqzpNWZuuYmvt3NbautbaupUrV863iwAAMGfzCsxVdf8MYfltrbV3jcW3VNX+4/r9k9w6lm9KsmZi89VJbpxP+wAAsNDm8y0ZleSvklzZWvufE6suTnLiuHxikvdMlB9fVbtW1QFJDkpy+VzbBwCAxbBiHtv+eJLnJvlsVX16LHtZkrOTXFRVJye5PslxSdJau6KqLkryuQzfsHFqa+2eebQPAAALbs6BubX20Uw/LzlJDp9hm7OSnDXXNgEAYLH5pT8AAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoENgBgCADoEZAAA6BGYAAOgQmAEAoGPRA3NVHVVVV1XVNVV1+mK3DwAA22NRA3NV7ZLkz5P8TJJDkpxQVYcsZh8AAGB7LPYI82FJrmmtfbG19u0kFyQ5ZpH7AAAA22yxA/OqJDdMPN40lgEAwE5pxSK3V9OUta0qVZ2S5JTx4R1VddWC9orFtk+Sryx1J5aj+sOl7gE7Ce+xBeI9xgTvswWyhO+zR860YrED86YkayYer05y49RKrbVzk5y7WJ1icVXVhtbauqXuByxX3mOw8LzP7lsWe0rG/0tyUFUdUFUPSHJ8kosXuQ8AALDNFnWEubV2d1X9RpJ/TLJLkje21q5YzD4AAMD2WOwpGWmtvTfJexe7XXYqptvAwvIeg4XnfXYfUq1tdc0dAAAw8tPYAADQITAzq6p6WFVdUFXXVtXnquq9VfXDVfWvU+qdWVW/PS6/uaqeOS5fWlUbJuqtq6pLx+X1VfWNqvr0xO2Icd3vVtUVVfWZsfzJE/u7aqL+OxfppYBFUVV3jPdrq6pV1W9OrPuzqjppXH5zVW2sqn+pqi9U1flVtWqi7nVVtc/E458f9/eYibJuG7Bcjef9WyYer6iqzVX1d1PqvaeqPjal7Hv/302z35VV9Z2q+rXx8SlVdeHE+oeM/58esGOfEQtJYKarqirJ3ya5tLX26NbaIUlelmS/7dzVvlX1MzOs+6fW2uMnbh+sqqcm+bkkT2yt/WiSI3LvH735xYn6z9zOvsAPkluTvGj8ZqHpvKS19rgkByf5VJIPd+qekOSjGb6haHvagOXoziSPraoHjo9/OsmXJytU1Z5Jnphkz+0IuMcl+XiG91uS/GWS1VsGg5K8MsOXHmycR99ZZAIzs3l6ku+01l6/paC19uncO7xuiz9K8vLtqL9/kq+01u4a2/xKa22r7+yG+4DNSS5JcmKvUhu8JsnNSbb647Sq9kjy40lOztaBeZvagGXoH5IcPS6fkOQdU9b/QpL/neSCbP2+mckJSU7LEJJXteFisecneW1VrUtyeIb/E/kBIjAzm8cm+cQM6x49OZUiya939vOxJHdV1dOnWfefpkzJeHSS9ydZM37MfE5V/eSUbd42Ud8/PCx3Zyc5rap22Ya6n0zymGnKj03yvtbaF5J8raqeOI82YLm4IMnxVbVbkh9NctmU9VtC9Dvy/RHjGVXVmiQPa61dnuSiJM9KktbaZzJ8pe4lSV7YWvv2DnsGLAqBmfm4dnIqRZLXz1L/1Zl+lHnqlIxrW2t3JPmxDD+RvjnJhVPmVE5OyXjJDngusNMaP7q9PMmzt6F6zVB+QoZwkPH+Xv/5b2cbsCyMQXZthvfDvb7ytqr2S3Jgko+Of2jeXVWPnWWXx2cIysnW77M/T/Ll1tqHd0DXWWQCM7O5IkNwnbfW2oeS7JbkKdtY/57W2qWttVck+Y0MH43BfdXvJ3lpZv93+wlJrpwsqKqHJvmpJG+oquuSvCTJs8ZrFObSBiwnFyf542w9HeNZSfZKsnF836zN7NMyTkhy0lj/4iSPq6qDxnXfHW/8APKPIrP5UJJdq+p5Wwqq6klJHjnH/Z2V5Hdmq1RVB0/8I5Mkj0/ypTm2CT/wWmufT/K5DBfDbqUGL8ww//99U1Y/M8n5rbVHttbWttbWJNmY5Gnb0wYsU29M8srW2menlJ+Q5KjxPbM2w+DRjIG5qg5O8qDW2qqJbf6gtw0/OARmusaLFX4+yU+PX4NzRZIzk8zpArzxlx43TymeOof5mUn2SHJeDV9j95kkh4ztbjE5h/mDc+kL/AA6K8nqKWV/VFX/kuQLSZ6U5OnTzI88IcO33Uz6m0w//WK6NmDZaq1taq39yWRZVa1N8ogM33axpd7GJLdv+YrTJC+vqk1bbpn5fTbr3Gd2fn7pDwAAOowwAwBAh8AMAAAdAjMAAHQIzAAA0CEwAwBAh8AMAAAdAjMAAHQIzAAA0PH/AdC9P09wBgiuAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
    " + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "# Subgroup Disparity Test\n", + "\n", + "sgd_test_gender = SubgroupDisparity(\n", + " attr='sex',\n", + " metric='mse',\n", + " method='ratio',\n", + " threshold=1.1,\n", + ")\n", + "sgd_test_gender.run(output)\n", + "sgd_test_gender.plot(alpha=0.05) # default alpha argument shows 95% C.I bands\n", + "\n", + "sgd_test_race = SubgroupDisparity(\n", + " attr='race',\n", + " metric='mse',\n", + " method='ratio',\n", + " threshold=1.1,\n", + ")\n", + "sgd_test_race.run(output)\n", + "sgd_test_race.plot(alpha=0.05) # default alpha argument shows 95% C.I bands" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bootstrap model card from tally form and scaffold assets\n", + "We can add the quantitative analysis, explainability analysis and fairness analysis sections to a bootstrap model card for convenience. In this example, we use an existing model card which we created from the tally form response.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Convert form response to model card protobuf\n", + "pb = tally_form_to_mc(\"sample-form-response-speech-to-text.json\")\n", + "\n", + "# Initialize the mct and scaffold using the existing protobuf\n", + "mct = mctlib.ModelCardToolkit(output_dir = \"model_card_output\", file_name=\"speech_to_text_example\")\n", + "mc = mct.scaffold_assets(proto=pb)\n", + "mc.model_details.name = \"Speech to Text Model\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Convert test objects to a model-card-compatible format" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# init model card test objects\n", + "mc_threshold_test_gender = mctlib.Test()\n", + "mc_threshold_test_race = mctlib.Test()\n", + "mc_sgd_test_race = mctlib.Test()\n", + "mc_sgd_test_gender = mctlib.Test()\n", + "\n", + "\n", + "# assign tests to them\n", + "mc_threshold_test_gender.read_model_test(threshold_test_gender)\n", + "mc_threshold_test_race.read_model_test(threshold_test_race)\n", + "mc_sgd_test_race.read_model_test(sgd_test_race)\n", + "mc_sgd_test_gender.read_model_test(sgd_test_gender)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "# Add quantitative analysis\n", + "\n", + "# Create 3 PerformanceMetric to store our results\n", + "mc.quantitative_analysis.performance_metrics = [mctlib.PerformanceMetric() for i in range(0, 3)]\n", + "mc.quantitative_analysis.performance_metrics[0].type = \"MSE\"\n", + "mc.quantitative_analysis.performance_metrics[0].value = str(mse_test)\n", + "mc.quantitative_analysis.performance_metrics[0].slice = \"Test Set\"\n", + "\n", + "mc.quantitative_analysis.performance_metrics[1].type = \"MAE\"\n", + "mc.quantitative_analysis.performance_metrics[1].value = str(mae_test)\n", + "mc.quantitative_analysis.performance_metrics[1].slice = \"Test Set\"\n", + "mc.quantitative_analysis.performance_metrics[1].graphics.description = (\n", + " 'About 65% of the words are correctly transcribed')\n", + "mc.quantitative_analysis.performance_metrics[1].graphics.collection = [\n", + " mctlib.Graphic(image=word_match_plot)\n", + "]\n", + "\n", + "mc.quantitative_analysis.performance_metrics[2].type = \"MSE across sensitive groups\"\n", + "mc.quantitative_analysis.performance_metrics[2].tests = [mc_threshold_test_gender, mc_threshold_test_race]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "# Add the fairness requirement for subgroup disparity\n", + "mc.fairness_analysis.fairness_reports[0].tests = [mc_sgd_test_race, mc_sgd_test_gender]\n", + "\n", + "\n", + "mct.update_model_card(mc)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Model Card Display" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " Model Card for Speech to Text Model\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "

    \n", + "\n", + " Model Card for Speech to Text Model\n", + "\n", + "

    \n", + "\n", + "
    \n", + "\n", + "
    \n", + "\n", + "

    Model Details

    \n", + "\n", + "

    Overview

    \n", + "\n", + " Using an automated transcription tool, the government want to transcribe audio files recorded in trials for evidential purposes. The tool should be able to capture the various spoken accents and transcribe into text accurately so as not to bias against the defendant. The tool chosen here will be Google's Speech to Text Model evaluated on a set of audio files by National Speech Corpus.\n", + "\n", + "

    Version

    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    name: v1
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + "

    Owners

    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
  • Timothy, Product Owner(s)
  • \n", + "\n", + " \n", + "\n", + "
  • Swan, Model Developer(s)
  • \n", + "\n", + " \n", + "\n", + "
  • Jason, Reviewer(s)
  • \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "

    Regulatory requirements

    \n", + "\n", + " MAS Fairness, Ethics, Accountability and Transparency (FEAT) principles\n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "

    Considerations

    \n", + "\n", + " \n", + "\n", + "

    Intended Users

    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
      \n", + "\n", + " \n", + "\n", + "
    • Judical System and Defendant
    • \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + "

    Use Cases

    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
      \n", + "\n", + " \n", + "\n", + "
    • Automate transcriptions and reduce human error and manpower in making court decisions
    • \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "

    Fairness Considerations

    \n", + "\n", + "
      \n", + "\n", + "
    • \n", + "\n", + "
      Group at risk: gender and race
      \n", + "\n", + "
      Benefits: Reduce manpower and human error on transcription services. A more accurate model than current system will aid the judicial system in making better court decisions
      \n", + "\n", + "
      Harms: Incorrectly transcribed/interpreted texts can impact on the defendant/ruling negatively.
      \n", + "\n", + "
      Mitigation Strategy: Improve upon the existing model by further training and enhancements
      \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "

    Datasets

    \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "
    \n", + "\n", + "

    National Speech Corpus Dataset

    \n", + "\n", + "

    Dataset taken from https://www.imda.gov.sg/programme-listing/digital-services-lab/national-speech-corpus

    \n", + "\n", + " \n", + "\n", + "

    Sensitive data

    \n", + "\n", + "
      \n", + "\n", + " \n", + "\n", + "
    • gender
    • \n", + "\n", + " \n", + "\n", + "
    • race
    • \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "

    Sensitive data used in model

    \n", + "\n", + "
      \n", + "\n", + " \n", + "\n", + "
    • N.A. (Protected attributes are not trained in the speech to text model)
    • \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "

    Justification

    \n", + "\n", + "

    N.A.

    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "

    Quantitative Analysis

    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "

    MSE - 712.513 (Test Set)

    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "

    MAE - 26.003 (Test Set)

    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "
    About 65% of the words are correctly transcribed
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " None\n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "

    MSE across sensitive groups

    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "

    Min Max Threshold Test

    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Description: \n", + "\n", + " Test if the mae of the subgroups within sex \n", + "is lower than the threshold of 27.\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Threshold: \n", + "\n", + " 27 \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Result: \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    maepassed\r", + "
    sex_F26.143True\r", + "
    sex_M25.891True\r", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    Passed
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Mean Absolute Error across sex subgroups\n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "

    Min Max Threshold Test

    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Description: \n", + "\n", + " Test if the mae of the subgroups within race \n", + "is lower than the threshold of 27.\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Threshold: \n", + "\n", + " 27 \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Result: \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    maepassed\r", + "
    race_CHINESE27.141False\r", + "
    race_INDIAN25.404True\r", + "
    race_MALAY25.47True\r", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    Failed
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Mean Absolute Error across race subgroups\n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "

    Fairness Analysis

    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "

    Equal mean squared error

    \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Segment: \n", + "\n", + " Gender and race \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Description: \n", + "\n", + " Disparity ratio of MSE of any 2 bins in the respective attribute should not be more than a factor of 1.1 \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "

    Subgroup Disparity Test

    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Description: \n", + "\n", + " Test if the maximum ratio of the mean squared error of any 2\n", + "groups within race attribute exceeds 1.1. To\n", + "pass, this value cannot exceed the threshold. \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Threshold: \n", + "\n", + " 1.1 \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Result: \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    race_mse_max_ratio\r", + "
    01.136\r", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    Failed
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Mean Squared Error across race subgroups\n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "

    Subgroup Disparity Test

    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Description: \n", + "\n", + " Test if the maximum ratio of the mean squared error of any 2\n", + "groups within sex attribute exceeds 1.1. To\n", + "pass, this value cannot exceed the threshold. \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Threshold: \n", + "\n", + " 1.1 \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Result: \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    sex_mse_max_ratio\r", + "
    01.023\r", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    Passed
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " Mean Squared Error across sex subgroups\n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
    \n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Export to html\n", + "html = mct.export_format(output_file=\"speech_to_text_example.html\")\n", + "display.display(display.HTML(html))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "veritas", + "language": "python", + "name": "veritas" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.12" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": {}, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}