Skip to content

Commit b2f18ef

Browse files
ZrahayZrahay
committed
BUG: Validate numeric_only parameter in groupby aggregations
- Add type check for numeric_only parameter in _cython_agg_general - Raise ValueError if numeric_only is not a boolean - Add test case for validation - Closes #62778
2 parents 52e07b5 + ea75dd7 commit b2f18ef

File tree

7 files changed

+596
-37
lines changed

7 files changed

+596
-37
lines changed

‎doc/source/whatsnew/v3.0.0.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,7 @@ Other enhancements
219219
- Added support to read and write from and to Apache Iceberg tables with the new :func:`read_iceberg` and :meth:`DataFrame.to_iceberg` functions (:issue:`61383`)
220220
- Errors occurring during SQL I/O will now throw a generic :class:`.DatabaseError` instead of the raw Exception type from the underlying driver manager library (:issue:`60748`)
221221
- Implemented :meth:`Series.str.isascii` and :meth:`Series.str.isascii` (:issue:`59091`)
222+
- Improve error reporting through outputting the first few duplicates when :func:`merge` validation fails (:issue:`62742`)
222223
- Improve the resulting dtypes in :meth:`DataFrame.where` and :meth:`DataFrame.mask` with :class:`ExtensionDtype` ``other`` (:issue:`62038`)
223224
- Improved deprecation message for offset aliases (:issue:`60820`)
224225
- Many type aliases are now exposed in the new submodule :py:mod:`pandas.api.typing.aliases` (:issue:`55231`)

‎pandas/_libs/index.pyx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,9 @@ cdef class IndexEngine:
321321
if is_strict_monotonic:
322322
self.unique = 1
323323
self.need_unique_check = 0
324+
elif self.monotonic_inc == 1 or self.monotonic_dec == 1:
325+
self.unique = 0
326+
self.need_unique_check = 0
324327

325328
cdef _call_monotonic(self, values):
326329
return algos.is_monotonic(values, timelike=False)

‎pandas/core/frame.py‎

Lines changed: 65 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3293,28 +3293,71 @@ def to_html(
32933293
Examples
32943294
--------
32953295
>>> df = pd.DataFrame(data={"col1": [1, 2], "col2": [4, 3]})
3296-
>>> html_string = '''<table border="1" class="dataframe">
3297-
... <thead>
3298-
... <tr style="text-align: right;">
3299-
... <th></th>
3300-
... <th>col1</th>
3301-
... <th>col2</th>
3302-
... </tr>
3303-
... </thead>
3304-
... <tbody>
3305-
... <tr>
3306-
... <th>0</th>
3307-
... <td>1</td>
3308-
... <td>4</td>
3309-
... </tr>
3310-
... <tr>
3311-
... <th>1</th>
3312-
... <td>2</td>
3313-
... <td>3</td>
3314-
... </tr>
3315-
... </tbody>
3316-
... </table>'''
3317-
>>> assert html_string == df.to_html()
3296+
>>> html_string = df.to_html()
3297+
>>> print(html_string)
3298+
<table border="1" class="dataframe">
3299+
<thead>
3300+
<tr style="text-align: right;">
3301+
<th></th>
3302+
<th>col1</th>
3303+
<th>col2</th>
3304+
</tr>
3305+
</thead>
3306+
<tbody>
3307+
<tr>
3308+
<th>0</th>
3309+
<td>1</td>
3310+
<td>4</td>
3311+
</tr>
3312+
<tr>
3313+
<th>1</th>
3314+
<td>2</td>
3315+
<td>3</td>
3316+
</tr>
3317+
</tbody>
3318+
</table>
3319+
3320+
HTML output
3321+
3322+
+----+-----+-----+
3323+
| |col1 |col2 |
3324+
+====+=====+=====+
3325+
|0 |1 |4 |
3326+
+----+-----+-----+
3327+
|1 |2 |3 |
3328+
+----+-----+-----+
3329+
3330+
>>> df = pd.DataFrame(data={"col1": [1, 2], "col2": [4, 3]})
3331+
>>> html_string = df.to_html(index=False)
3332+
>>> print(html_string)
3333+
<table border="1" class="dataframe">
3334+
<thead>
3335+
<tr style="text-align: right;">
3336+
<th>col1</th>
3337+
<th>col2</th>
3338+
</tr>
3339+
</thead>
3340+
<tbody>
3341+
<tr>
3342+
<td>1</td>
3343+
<td>4</td>
3344+
</tr>
3345+
<tr>
3346+
<td>2</td>
3347+
<td>3</td>
3348+
</tr>
3349+
</tbody>
3350+
</table>
3351+
3352+
HTML output
3353+
3354+
+-----+-----+
3355+
|col1 |col2 |
3356+
+=====+=====+
3357+
|1 |4 |
3358+
+-----+-----+
3359+
|2 |3 |
3360+
+-----+-----+
33183361
"""
33193362
if justify is not None and justify not in fmt.VALID_JUSTIFY_PARAMETERS:
33203363
raise ValueError("Invalid value for justify parameter")

‎pandas/core/reshape/merge.py‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1954,42 +1954,62 @@ def _validate_left_right_on(self, left_on, right_on):
19541954
def _validate_validate_kwd(self, validate: str) -> None:
19551955
# Check uniqueness of each
19561956
if self.left_index:
1957-
left_unique = self.orig_left.index.is_unique
1957+
left_join_index = self.orig_left.index
1958+
left_unique = left_join_index.is_unique
19581959
else:
1959-
left_unique = MultiIndex.from_arrays(self.left_join_keys).is_unique
1960+
left_join_index = MultiIndex.from_arrays(self.left_join_keys)
1961+
left_unique = left_join_index.is_unique
19601962

19611963
if self.right_index:
1964+
right_join_index = self.orig_right.index
19621965
right_unique = self.orig_right.index.is_unique
19631966
else:
1964-
right_unique = MultiIndex.from_arrays(self.right_join_keys).is_unique
1967+
right_join_index = MultiIndex.from_arrays(self.right_join_keys)
1968+
right_unique = right_join_index.is_unique
1969+
1970+
def left_error_msg(x: Index) -> str:
1971+
name = self.left_on if not self.left_index else lib.no_default
1972+
msg = x[x.duplicated()][:5].to_frame(name=name).to_string(index=False)
1973+
return f"\nDuplicates in left:\n {msg} ..."
1974+
1975+
def right_error_msg(x: Index) -> str:
1976+
name = self.right_on if not self.right_index else lib.no_default
1977+
msg = x[x.duplicated()][:5].to_frame(name=name).to_string(index=False)
1978+
return f"\nDuplicates in right:\n {msg} ..."
19651979

19661980
# Check data integrity
19671981
if validate in ["one_to_one", "1:1"]:
19681982
if not left_unique and not right_unique:
19691983
raise MergeError(
19701984
"Merge keys are not unique in either left "
1971-
"or right dataset; not a one-to-one merge"
1985+
"or right dataset; not a one-to-one merge."
1986+
f"{left_error_msg(left_join_index)}"
1987+
f"{right_error_msg(right_join_index)}"
19721988
)
19731989
if not left_unique:
19741990
raise MergeError(
19751991
"Merge keys are not unique in left dataset; not a one-to-one merge"
1992+
f"{left_error_msg(left_join_index)}"
19761993
)
19771994
if not right_unique:
19781995
raise MergeError(
19791996
"Merge keys are not unique in right dataset; not a one-to-one merge"
1997+
f"{right_error_msg(right_join_index)}"
19801998
)
19811999

19822000
elif validate in ["one_to_many", "1:m"]:
19832001
if not left_unique:
19842002
raise MergeError(
19852003
"Merge keys are not unique in left dataset; not a one-to-many merge"
2004+
f"{left_error_msg(left_join_index)}"
19862005
)
19872006

19882007
elif validate in ["many_to_one", "m:1"]:
19892008
if not right_unique:
19902009
raise MergeError(
19912010
"Merge keys are not unique in right dataset; "
1992-
"not a many-to-one merge"
2011+
"not a many-to-one merge\n"
2012+
f"{right_error_msg(right_join_index)}"
19932013
)
19942014

19952015
elif validate in ["many_to_many", "m:m"]:

0 commit comments

Comments
 (0)