def test_consumption_response(cps_subsample):
    consump = Consumption()
    mpc = 0.5
    consumption_response = {'MPC_e20400': {2013: mpc}}
    consump.update_consumption(consumption_response)
    # test incorrect call to response method
    with pytest.raises(ValueError):
        consump.response(list(), 1)
    # test correct call to response method
    rec = Records.cps_constructor(data=cps_subsample)
    pre = copy.deepcopy(rec.e20400)
    consump.response(rec, 1.0)
    post = rec.e20400
    actual_diff = post - pre
    expected_diff = np.ones(rec.array_length) * mpc
    assert np.allclose(actual_diff, expected_diff)
    # compute earnings mtr with no consumption response
    rec = Records.cps_constructor(data=cps_subsample)
    ided0 = copy.deepcopy(rec.e20400)
    calc0 = Calculator(policy=Policy(), records=rec, consumption=None)
    (mtr0_ptax, mtr0_itax, _) = calc0.mtr(variable_str='e00200p',
                                          wrt_full_compensation=False)
    assert np.allclose(calc0.array('e20400'), ided0)
    # compute earnings mtr with consumption response
    calc1 = Calculator(policy=Policy(), records=rec, consumption=consump)
    mtr1_ptax, mtr1_itax, _ = calc1.mtr(variable_str='e00200p',
                                        wrt_full_compensation=False)
    assert np.allclose(calc1.array('e20400'), ided0)
    # confirm that payroll mtr values are no different
    assert np.allclose(mtr1_ptax, mtr0_ptax)
    # confirm that all mtr with cons-resp are no greater than without cons-resp
    assert np.all(np.less_equal(np.around(mtr1_itax, decimals=5),
                                np.around(mtr0_itax, decimals=5)))
    # confirm that some mtr with cons-resp are less than without cons-resp
    assert np.any(np.less(mtr1_itax, mtr0_itax))

def test_calc_and_used_vars(tests_path):
    """
    Runs two kinds of tests on variables used in the calcfunctions.py file:

    (1) Checks that each var in Records.CALCULATED_VARS is actually calculated

    If test (1) fails, a variable in Records.CALCULATED_VARS was not
    calculated in any function in the calcfunctions.py file.  With the
    exception of a few variables listed in this test, all
    Records.CALCULATED_VARS must be calculated in the calcfunctions.py file.

    (2) Check that each variable that is calculated in a function and
    returned by that function is an argument of that function.
    """
    # pylint: disable=too-many-locals
    funcpath = os.path.join(tests_path, '..', 'calcfunctions.py')
    gfd = GetFuncDefs()
    fnames, fargs, cvars, rvars = gfd.visit(ast.parse(open(funcpath).read()))
    # Test (1):
    # .. create set of vars that are actually calculated in calcfunctions.py
    all_cvars = set()
    for fname in fnames:
        if fname == 'BenefitSurtax':
            continue  # because BenefitSurtax is not really a function
        all_cvars.update(set(cvars[fname]))
    # .. add to all_cvars set variables calculated in Records class
    all_cvars.update(set(['num', 'sep', 'exact']))
    # .. add to all_cvars set variables calculated elsewhere
    all_cvars.update(set(['mtr_paytax', 'mtr_inctax']))
    all_cvars.update(set(['benefit_cost_total', 'benefit_value_total']))
    # .. check that each var in Records.CALCULATED_VARS is in the all_cvars set
    Records.read_var_info()
    found_error1 = False
    if not Records.CALCULATED_VARS <= all_cvars:
        msg1 = ('all Records.CALCULATED_VARS not calculated '
                'in calcfunctions.py\n')
        for var in Records.CALCULATED_VARS - all_cvars:
            found_error1 = True
            msg1 += 'VAR NOT CALCULATED: {}\n'.format(var)
    # Test (2):
    faux_functions = ['EITCamount', 'ComputeBenefit', 'BenefitPrograms',
                      'BenefitSurtax', 'BenefitLimitation']
    found_error2 = False
    msg2 = 'calculated & returned variables are not function arguments\n'
    for fname in fnames:
        if fname in faux_functions:
            continue  # because fname is not a genuine function
        crvars_set = set(cvars[fname]) & set(rvars[fname])
        if not crvars_set <= set(fargs[fname]):
            found_error2 = True
            for var in crvars_set - set(fargs[fname]):
                msg2 += 'FUNCTION,VARIABLE: {} {}\n'.format(fname, var)
    # Report errors for the two tests:
    if found_error1 and found_error2:
        raise ValueError('{}\n{}'.format(msg1, msg2))
    if found_error1:
        raise ValueError(msg1)
    if found_error2:
        raise ValueError(msg2)

def test_validity_of_name_lists():
    assert len(DIST_TABLE_COLUMNS) == len(DIST_TABLE_LABELS)
    Records.read_var_info()
    assert set(DIST_VARIABLES).issubset(Records.CALCULATED_VARS | {'s006'})
    extra_vars_set = set(['num_returns_StandardDed',
                          'num_returns_ItemDed',
                          'num_returns_AMT'])
    assert (set(DIST_TABLE_COLUMNS) - set(DIST_VARIABLES)) == extra_vars_set

def test_correct_Records_instantiation():
    rec1 = Records(data=TAXDATA, blowup_factors=None, weights=WEIGHTS)
    assert rec1
    assert np.all(rec1.MARS != 0)
    assert rec1.current_year == Records.PUF_YEAR
    sum_e00200_in_puf_year = rec1.e00200.sum()
    rec1.set_current_year(Records.PUF_YEAR + 1)
    sum_e00200_in_puf_year_plus_one = rec1.e00200.sum()
    assert sum_e00200_in_puf_year_plus_one == sum_e00200_in_puf_year
    bf_df = pd.read_csv(Records.BLOWUP_FACTORS_PATH)
    rec2 = Records(data=TAXDATA, blowup_factors=bf_df, weights=None)
    assert rec2
    assert np.all(rec2.MARS != 0)
    assert rec2.current_year == Records.PUF_YEAR

def test_correct_Records_instantiation_sample(puf_1991, weights_1991):
    sample = puf_1991.sample(frac=0.10)
    rec1 = Records(data=sample, blowup_factors=None, weights=weights_1991)
    assert rec1
    assert np.all(rec1.MARS != 0)
    assert rec1.current_year == Records.PUF_YEAR
    sum_e00200_in_puf_year = rec1.e00200.sum()
    rec1.set_current_year(Records.PUF_YEAR + 1)
    sum_e00200_in_puf_year_plus_one = rec1.e00200.sum()
    assert sum_e00200_in_puf_year_plus_one == sum_e00200_in_puf_year
    bf_df = pd.read_csv(Records.BLOWUP_FACTORS_PATH)
    rec2 = Records(data=sample, blowup_factors=bf_df, weights=None)
    assert rec2
    assert np.all(rec2.MARS != 0)
    assert rec2.current_year == Records.PUF_YEAR

def test_ID_HC_vs_BS(cps_subsample):
    """
    Test that complete haircut of itemized deductions produces same
    results as a 100% benefit surtax with no benefit deduction.
    """
    recs = Records.cps_constructor(data=cps_subsample)
    # specify complete-haircut reform policy and Calculator object
    hc_reform = {2013: {'_ID_Medical_hc': [1.0],
                        '_ID_StateLocalTax_hc': [1.0],
                        '_ID_RealEstate_hc': [1.0],
                        '_ID_Casualty_hc': [1.0],
                        '_ID_Miscellaneous_hc': [1.0],
                        '_ID_InterestPaid_hc': [1.0],
                        '_ID_Charity_hc': [1.0]}}
    hc_policy = Policy()
    hc_policy.implement_reform(hc_reform)
    hc_calc = Calculator(policy=hc_policy, records=recs)
    hc_calc.calc_all()
    hc_taxes = hc_calc.dataframe(['iitax', 'payrolltax'])
    del hc_calc
    # specify benefit-surtax reform policy and Calculator object
    bs_reform = {2013: {'_ID_BenefitSurtax_crt': [0.0],
                        '_ID_BenefitSurtax_trt': [1.0]}}
    bs_policy = Policy()
    bs_policy.implement_reform(bs_reform)
    bs_calc = Calculator(policy=bs_policy, records=recs)
    bs_calc.calc_all()
    bs_taxes = bs_calc.dataframe(['iitax', 'payrolltax'])
    del bs_calc
    # compare calculated taxes generated by the two reforms
    assert np.allclose(hc_taxes['payrolltax'], bs_taxes['payrolltax'])
    assert np.allclose(hc_taxes['iitax'], bs_taxes['iitax'])

def test_distribution_tables(cps_subsample):
    """
    Test distribution_tables method.
    """
    pol = Policy()
    recs = Records.cps_constructor(data=cps_subsample)
    calc1 = Calculator(policy=pol, records=recs)
    assert calc1.current_year == 2014
    calc1.calc_all()
    dt1, dt2 = calc1.distribution_tables(None, 'weighted_deciles')
    assert isinstance(dt1, pd.DataFrame)
    assert dt2 is None
    dt1, dt2 = calc1.distribution_tables(calc1, 'weighted_deciles')
    assert isinstance(dt1, pd.DataFrame)
    assert isinstance(dt2, pd.DataFrame)
    reform = {2014: {'_UBI_u18': [1000],
                     '_UBI_1820': [1000],
                     '_UBI_21': [1000]}}
    pol.implement_reform(reform)
    assert not pol.parameter_errors
    calc2 = Calculator(policy=pol, records=recs)
    calc2.calc_all()
    dt1, dt2 = calc1.distribution_tables(calc2, 'weighted_deciles')
    assert isinstance(dt1, pd.DataFrame)
    assert isinstance(dt2, pd.DataFrame)

def test_n65(cps_subsample):
    """
    Test n65 method.
    """
    recs = Records.cps_constructor(data=cps_subsample)
    calc = Calculator(policy=Policy(), records=recs)
    assert calc.n65().sum() > 1500

def test_write_graph_file(cps_subsample):
    recs = Records.cps_constructor(data=cps_subsample)
    calc = Calculator(policy=Policy(), records=recs)
    mtr = 0.20 * np.ones_like(cps_subsample['e00200'])
    vdf = calc.dataframe(['s006', 'e00200', 'c00100'])
    vdf['mtr1'] = mtr
    vdf['mtr2'] = mtr
    gdata = mtr_graph_data(vdf, calc.current_year, mtr_measure='ptax',
                           alt_e00200p_text='Taxpayer Earnings',
                           income_measure='agi',
                           dollar_weighting=False)
    gplot = xtr_graph_plot(gdata)
    assert gplot
    htmlfname = temporary_filename(suffix='.html')
    try:
        write_graph_file(gplot, htmlfname, 'title')
    except Exception:  # pylint: disable=broad-except
        if os.path.isfile(htmlfname):
            try:
                os.remove(htmlfname)
            except OSError:
                pass  # sometimes we can't remove a generated temporary file
        assert 'write_graph_file()_ok' == 'no'
    # if try was successful, try to remove the file
    if os.path.isfile(htmlfname):
        try:
            os.remove(htmlfname)
        except OSError:
            pass  # sometimes we can't remove a generated temporary file

def test_mtr_graph_data(cps_subsample):
    recs = Records.cps_constructor(data=cps_subsample)
    calc = Calculator(policy=Policy(), records=recs)
    year = calc.current_year
    with pytest.raises(ValueError):
        mtr_graph_data(None, year, mars='bad',
                       income_measure='agi',
                       dollar_weighting=True)
    with pytest.raises(ValueError):
        mtr_graph_data(None, year, mars=0,
                       income_measure='expanded_income',
                       dollar_weighting=True)
    with pytest.raises(ValueError):
        mtr_graph_data(None, year, mars=list())
    with pytest.raises(ValueError):
        mtr_graph_data(None, year, mars='ALL', mtr_variable='e00200s')
    with pytest.raises(ValueError):
        mtr_graph_data(None, year, mtr_measure='badtax')
    with pytest.raises(ValueError):
        mtr_graph_data(None, year, income_measure='badincome')
    mtr = 0.20 * np.ones_like(cps_subsample['e00200'])
    vdf = calc.dataframe(['s006', 'MARS', 'e00200'])
    vdf['mtr1'] = mtr
    vdf['mtr2'] = mtr
    vdf = vdf[vdf['MARS'] == 1]
    gdata = mtr_graph_data(vdf, year, mars=1,
                           mtr_wrt_full_compen=True,
                           income_measure='wages',
                           dollar_weighting=True)
    assert isinstance(gdata, dict)

def test_itax_compare(tests_path, using_puf, puf_fullsample, cps_fullsample):
    """
    Conduct income tax comparisons using ITAX data.
    """
    using_puf_adjust_ratios = True
    # generate 2015 estimates by AGI category using Tax-Calculator
    if using_puf:
        if using_puf_adjust_ratios:
            recs = Records(data=puf_fullsample)
        else:
            recs = Records(data=puf_fullsample, adjust_ratios=None)
    else:
        recs = Records.cps_constructor(data=cps_fullsample)
    calc = Calculator(policy=Policy(), records=recs, verbose=False)
    calc.advance_to_year(2015)
    calc.calc_all()
    # open actual output file
    if using_puf:
        afilename = os.path.join(tests_path, 'cmpi_puf_actual.txt')
    else:
        afilename = os.path.join(tests_path, 'cmpi_cps_actual.txt')
    afile = open(afilename, 'w')
    # write compare results to afile
    for cname in sorted(ITAX.keys()):
        comparison(cname, calc, ITAX, afile)
    # close actual output file
    afile.close()
    # check for differences between actual and expect output files
    efilename = afilename.replace('actual', 'expect')
    differences(afilename, efilename)

def test_dec_graph_plots(cps_subsample):
    pol = Policy()
    rec = Records.cps_constructor(data=cps_subsample)
    calc1 = Calculator(policy=pol, records=rec)
    year = 2020
    calc1.advance_to_year(year)
    reform = {
        'SS_Earnings_c': {year: 9e99},  # OASDI FICA tax on all earnings
        'FICA_ss_trt': {year: 0.107484}  # lower rate to keep revenue unchanged
    }
    pol.implement_reform(reform)
    calc2 = Calculator(policy=pol, records=rec)
    calc2.advance_to_year(year)
    assert calc1.current_year == calc2.current_year
    calc1.calc_all()
    calc2.calc_all()
    fig = calc1.decile_graph(calc2)
    assert fig
    dt1, dt2 = calc1.distribution_tables(calc2, 'weighted_deciles')
    dta = dec_graph_data(dt1, dt2, year,
                         include_zero_incomes=True,
                         include_negative_incomes=False)
    assert isinstance(dta, dict)
    dta = dec_graph_data(dt1, dt2, year,
                         include_zero_incomes=False,
                         include_negative_incomes=True)
    assert isinstance(dta, dict)
    dta = dec_graph_data(dt1, dt2, year,
                         include_zero_incomes=False,
                         include_negative_incomes=False)
    assert isinstance(dta, dict)

def test_make_calculator_with_multiyear_reform(cps_subsample):
    """
    Test Calculator class ctor with multi-year policy reform.
    """
    rec = Records.cps_constructor(data=cps_subsample)
    year = rec.current_year
    # create a Policy object and apply a policy reform
    pol = Policy()
    reform = {2015: {}, 2016: {}}
    reform[2015]['_II_em'] = [5000, 6000]  # reform values for 2015 and 2016
    reform[2015]['_II_em_cpi'] = False
    reform[2016]['_STD_Aged'] = [[1600, 1300, 1600, 1300, 1600]]
    pol.implement_reform(reform)
    # create a Calculator object using this policy-reform
    calc = Calculator(policy=pol, records=rec)
    # check that Policy object embedded in Calculator object is correct
    assert pol.num_years == Policy.DEFAULT_NUM_YEARS
    assert calc.current_year == year
    assert calc.policy_param('II_em') == 3950
    exp_II_em = [3900, 3950, 5000] + [6000] * (Policy.DEFAULT_NUM_YEARS - 3)
    assert np.allclose(calc.policy_param('_II_em'),
                       np.array(exp_II_em))
    calc.increment_year()
    calc.increment_year()
    assert calc.current_year == 2016
    assert np.allclose(calc.policy_param('STD_Aged'),
                       np.array([1600, 1300, 1600, 1300, 1600]))

def test_make_calculator_with_policy_reform(cps_subsample):
    """
    Test Calculator class ctor with policy reform.
    """
    rec = Records.cps_constructor(data=cps_subsample)
    year = rec.current_year
    # create a Policy object and apply a policy reform
    pol = Policy()
    reform = {2013: {'_II_em': [4000], '_II_em_cpi': False,
                     '_STD_Aged': [[1600, 1300, 1300, 1600, 1600]],
                     '_STD_Aged_cpi': False}}
    pol.implement_reform(reform)
    # create a Calculator object using this policy reform
    calc = Calculator(policy=pol, records=rec)
    # check that Policy object embedded in Calculator object is correct
    assert calc.current_year == year
    assert calc.policy_param('II_em') == 4000
    assert np.allclose(calc.policy_param('_II_em'),
                       np.array([4000] * Policy.DEFAULT_NUM_YEARS))
    exp_STD_Aged = [[1600, 1300, 1300,
                     1600, 1600]] * Policy.DEFAULT_NUM_YEARS
    assert np.allclose(calc.policy_param('_STD_Aged'),
                       np.array(exp_STD_Aged))
    assert np.allclose(calc.policy_param('STD_Aged'),
                       np.array([1600, 1300, 1300, 1600, 1600]))

def test_make_calculator_raises_on_no_policy(cps_subsample):
    """
    Test Calculator ctor error with no policy argument.
    """
    rec = Records.cps_constructor(data=cps_subsample)
    with pytest.raises(ValueError):
        Calculator(records=rec)

def test_make_calculator_increment_years_first(cps_subsample):
    """
    Test Calculator inflation indexing of policy parameters.
    """
    # pylint: disable=too-many-locals
    # create Policy object with policy reform
    pol = Policy()
    reform = {2015: {}, 2016: {}}
    std5 = 2000
    reform[2015]['_STD_Aged'] = [[std5, std5, std5, std5, std5]]
    reform[2015]['_II_em'] = [5000]
    reform[2016]['_II_em'] = [6000]
    reform[2016]['_II_em_cpi'] = False
    pol.implement_reform(reform)
    # create Calculator object with Policy object as modified by reform
    rec = Records.cps_constructor(data=cps_subsample)
    calc = Calculator(policy=pol, records=rec)
    # compare expected policy parameter values with those embedded in calc
    irates = pol.inflation_rates()
    syr = Policy.JSON_START_YEAR
    irate2015 = irates[2015 - syr]
    irate2016 = irates[2016 - syr]
    std6 = std5 * (1.0 + irate2015)
    std7 = std6 * (1.0 + irate2016)
    exp_STD_Aged = np.array([[1500, 1200, 1200, 1500, 1500],
                             [1550, 1200, 1200, 1550, 1550],
                             [std5, std5, std5, std5, std5],
                             [std6, std6, std6, std6, std6],
                             [std7, std7, std7, std7, std7]])
    act_STD_Aged = calc.policy_param('_STD_Aged')
    assert np.allclose(act_STD_Aged[:5], exp_STD_Aged)
    exp_II_em = np.array([3900, 3950, 5000, 6000, 6000])
    act_II_em = calc.policy_param('_II_em')
    assert np.allclose(act_II_em[:5], exp_II_em)

def test_blowup():
    pol1 = Policy()
    assert pol1.current_year == Policy.JSON_START_YEAR
    rec1 = Records(data=TAXDATA, weights=WEIGHTS)
    assert rec1.current_year == Records.PUF_YEAR
    calc1 = Calculator(policy=pol1, records=rec1, sync_years=True)
    assert calc1.records.current_year == Policy.JSON_START_YEAR
    pol2 = Policy()
    assert pol2.current_year == Policy.JSON_START_YEAR
    rec2 = Records(data=TAXDATA, weights=WEIGHTS)
    assert rec2.current_year == Records.PUF_YEAR
    rec2.set_current_year(Policy.JSON_START_YEAR)
    assert rec2.current_year == Policy.JSON_START_YEAR
    calc2 = Calculator(policy=pol2, records=rec2, sync_years=False)
    assert calc2.policy.current_year == Policy.JSON_START_YEAR
    assert calc2.records.current_year == Policy.JSON_START_YEAR

def test_diagnostic_table(cps_subsample):
    """
    Test diagnostic_table method.
    """
    recs = Records.cps_constructor(data=cps_subsample)
    calc = Calculator(policy=Policy(), records=recs)
    adt = calc.diagnostic_table(3)
    assert isinstance(adt, pd.DataFrame)

def test_dist_table_sum_row(cps_subsample):
    rec = Records.cps_constructor(data=cps_subsample)
    calc = Calculator(policy=Policy(), records=rec)
    calc.calc_all()
    tb1 = create_distribution_table(calc.distribution_table_dataframe(),
                                    'standard_income_bins', 'expanded_income')
    tb2 = create_distribution_table(calc.distribution_table_dataframe(),
                                    'soi_agi_bins', 'expanded_income')
    assert np.allclose(tb1[-1:], tb2[-1:])

def test_make_calculator_deepcopy(cps_subsample):
    """
    Test deepcopy of Calculator object.
    """
    pol = Policy()
    rec = Records.cps_constructor(data=cps_subsample)
    calc1 = Calculator(policy=pol, records=rec)
    calc2 = copy.deepcopy(calc1)
    assert isinstance(calc2, Calculator)