<?xml version="1.0" encoding="us-ascii"?>
<!--XBRL document created with Toppan Merrill Bridge  9.8.1.3 -->
<!--Based on XBRL 2.1-->
<!--Created on: 11/6/2020 4:03:20 PM-->
<!--Modified on: 11/6/2020 4:03:20 PM-->
<link:linkbase xmlns:link="http://www.xbrl.org/2003/linkbase" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xbrli="http://www.xbrl.org/2003/instance" xmlns:xlink="http://www.w3.org/1999/xlink" xsi:schemaLocation="http://www.xbrl.org/2003/linkbase http://www.xbrl.org/2003/xbrl-linkbase-2003-12-31.xsd">
  <roleRef xlink:type="simple" xlink:href="imh-20200930.xsd#StatementConsolidatedBalanceSheets" roleURI="http://www.impaccompanies.com/role/StatementConsolidatedBalanceSheets" xmlns="http://www.xbrl.org/2003/linkbase" />
  <roleRef xlink:type="simple" xlink:href="imh-20200930.xsd#StatementConsolidatedStatementsOfOperationsAndComprehensiveEarningsLoss" roleURI="http://www.impaccompanies.com/role/StatementConsolidatedStatementsOfOperationsAndComprehensiveEarningsLoss" xmlns="http://www.xbrl.org/2003/linkbase" />
  <roleRef xlink:type="simple" xlink:href="imh-20200930.xsd#StatementConsolidatedStatementsOfCashFlows" roleURI="http://www.impaccompanies.com/role/StatementConsolidatedStatementsOfCashFlows" xmlns="http://www.xbrl.org/2003/linkbase" />
  <roleRef xlink:type="simple" xlink:href="imh-20200930.xsd#DisclosureMortgageLoansHeldForSaleDetails" roleURI="http://www.impaccompanies.com/role/DisclosureMortgageLoansHeldForSaleDetails" xmlns="http://www.xbrl.org/2003/linkbase" />
  <roleRef xlink:type="simple" xlink:href="imh-20200930.xsd#DisclosureMortgageServicingRightsDetails" roleURI="http://www.impaccompanies.com/role/DisclosureMortgageServicingRightsDetails" xmlns="http://www.xbrl.org/2003/linkbase" />
  <roleRef xlink:type="simple" xlink:href="imh-20200930.xsd#DisclosureLeasesMaturitiesDetails" roleURI="http://www.impaccompanies.com/role/DisclosureLeasesMaturitiesDetails" xmlns="http://www.xbrl.org/2003/linkbase" />
  <roleRef xlink:type="simple" xlink:href="imh-20200930.xsd#DisclosureLeasesMaturitiesDetailsCal2" roleURI="http://www.impaccompanies.com/role/DisclosureLeasesMaturitiesDetailsCal2" xmlns="http://www.xbrl.org/2003/linkbase" />
  <roleRef xlink:type="simple" xlink:href="imh-20200930.xsd#DisclosureSecuritizedMortgageTrustsSecuritizedMortgageTrustAssetsDetails" roleURI="http://www.impaccompanies.com/role/DisclosureSecuritizedMortgageTrustsSecuritizedMortgageTrustAssetsDetails" xmlns="http://www.xbrl.org/2003/linkbase" />
  <roleRef xlink:type="simple" xlink:href="imh-20200930.xsd#DisclosureSecuritizedMortgageTrustsChangeInFairValueOfNetTrustAssetsDetails" roleURI="http://www.impaccompanies.com/role/DisclosureSecuritizedMortgageTrustsChangeInFairValueOfNetTrustAssetsDetails" xmlns="http://www.xbrl.org/2003/linkbase" />
  <roleRef xlink:type="simple" xlink:href="imh-20200930.xsd#DisclosureFairValueOfFinancialInstrumentsAssetsAndLiabilitiesMeasuredAtFairValueOnRecurringBasisDetails" roleURI="http://www.impaccompanies.com/role/DisclosureFairValueOfFinancialInstrumentsAssetsAndLiabilitiesMeasuredAtFairValueOnRecurringBasisDetails" xmlns="http://www.xbrl.org/2003/linkbase" />
  <roleRef xlink:type="simple" xlink:href="imh-20200930.xsd#DisclosureFairValueOfFinancialInstrumentsChangesInRecurringFairValueMeasurementsIncludedInEarningsDetails" roleURI="http://www.impaccompanies.com/role/DisclosureFairValueOfFinancialInstrumentsChangesInRecurringFairValueMeasurementsIncludedInEarningsDetails" xmlns="http://www.xbrl.org/2003/linkbase" />
  <roleRef xlink:type="simple" xlink:href="imh-20200930.xsd#DisclosureReconciliationOfLossEarningsPerCommonShareDetails" roleURI="http://www.impaccompanies.com/role/DisclosureReconciliationOfLossEarningsPerCommonShareDetails" xmlns="http://www.xbrl.org/2003/linkbase" />
  <roleRef xlink:type="simple" xlink:href="imh-20200930.xsd#DisclosureSegmentReportingDetails" roleURI="http://www.impaccompanies.com/role/DisclosureSegmentReportingDetails" xmlns="http://www.xbrl.org/2003/linkbase" />
  <roleRef xlink:type="simple" xlink:href="imh-20200930.xsd#DisclosureCommitmentsAndContingenciesCorporateOwnedLifeInsuranceTrustsDetails" roleURI="http://www.impaccompanies.com/role/DisclosureCommitmentsAndContingenciesCorporateOwnedLifeInsuranceTrustsDetails" xmlns="http://www.xbrl.org/2003/linkbase" />
  <calculationLink xlink:role="http://www.impaccompanies.com/role/StatementConsolidatedBalanceSheets" xlink:type="extended" xlink:title="00100 - Statement - CONSOLIDATED BALANCE SHEETS" xmlns="http://www.xbrl.org/2003/linkbase">
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_Assets" xlink:label="us-gaap_Assets" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CashAndCashEquivalentsAtCarryingValue" xlink:label="us-gaap_CashAndCashEquivalentsAtCarryingValue_637402753956477988" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_Assets" xlink:to="us-gaap_CashAndCashEquivalentsAtCarryingValue_637402753956477988" order="1" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RestrictedCashAndCashEquivalentsAtCarryingValue" xlink:label="us-gaap_RestrictedCashAndCashEquivalentsAtCarryingValue_637402753956487970" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_Assets" xlink:to="us-gaap_RestrictedCashAndCashEquivalentsAtCarryingValue_637402753956487970" order="2" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LoansReceivableHeldForSaleNetNotPartOfDisposalGroup" xlink:label="us-gaap_LoansReceivableHeldForSaleNetNotPartOfDisposalGroup_637402753956497945" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_Assets" xlink:to="us-gaap_LoansReceivableHeldForSaleNetNotPartOfDisposalGroup_637402753956497945" order="3" use="optional" weight="1" priority="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ServicingAssetAtFairValueAmount" xlink:label="us-gaap_ServicingAssetAtFairValueAmount_637402753956497945" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_Assets" xlink:to="us-gaap_ServicingAssetAtFairValueAmount_637402753956497945" order="4" use="optional" weight="1" priority="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_SecuritizedMortgageTrustAssets" xlink:label="imh_SecuritizedMortgageTrustAssets_637402753956497945" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_Assets" xlink:to="imh_SecuritizedMortgageTrustAssets_637402753956497945" order="5" use="optional" weight="1" priority="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OtherAssets" xlink:label="us-gaap_OtherAssets_637402753956497945" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_Assets" xlink:to="us-gaap_OtherAssets_637402753956497945" order="6" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LiabilitiesAndStockholdersEquity" xlink:label="us-gaap_LiabilitiesAndStockholdersEquity" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_Liabilities" xlink:label="us-gaap_Liabilities_637402753956587703" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_LiabilitiesAndStockholdersEquity" xlink:to="us-gaap_Liabilities_637402753956587703" order="1" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_WarehouseAgreementBorrowings" xlink:label="us-gaap_WarehouseAgreementBorrowings_637402753956587703" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_Liabilities_637402753956587703" xlink:to="us-gaap_WarehouseAgreementBorrowings_637402753956587703" order="1" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_MortgageServicingRightFinancingFacility" xlink:label="imh_MortgageServicingRightFinancingFacility_637402753956587703" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_Liabilities_637402753956587703" xlink:to="imh_MortgageServicingRightFinancingFacility_637402753956587703" order="2" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ConvertibleNotesPayable" xlink:label="us-gaap_ConvertibleNotesPayable_637402753956587703" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_Liabilities_637402753956587703" xlink:to="us-gaap_ConvertibleNotesPayable_637402753956587703" order="3" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OtherLongTermDebt" xlink:label="us-gaap_OtherLongTermDebt_637402753956587703" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_Liabilities_637402753956587703" xlink:to="us-gaap_OtherLongTermDebt_637402753956587703" order="4" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_SecuritizedMortgageTrustLiabilities" xlink:label="imh_SecuritizedMortgageTrustLiabilities_637402753956598018" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_Liabilities_637402753956587703" xlink:to="imh_SecuritizedMortgageTrustLiabilities_637402753956598018" order="5" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OtherLiabilities" xlink:label="us-gaap_OtherLiabilities_637402753956598018" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_Liabilities_637402753956587703" xlink:to="us-gaap_OtherLiabilities_637402753956598018" order="6" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_StockholdersEquity" xlink:label="us-gaap_StockholdersEquity_637402753956598018" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_LiabilitiesAndStockholdersEquity" xlink:to="us-gaap_StockholdersEquity_637402753956598018" order="2" use="optional" weight="1" priority="3" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PreferredStockValue" xlink:label="us-gaap_PreferredStockValue_637402753956598018" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_StockholdersEquity_637402753956598018" xlink:to="us-gaap_PreferredStockValue_637402753956598018" order="1" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CommonStockValue" xlink:label="us-gaap_CommonStockValue_637402753956598018" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_StockholdersEquity_637402753956598018" xlink:to="us-gaap_CommonStockValue_637402753956598018" order="2" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AdditionalPaidInCapital" xlink:label="us-gaap_AdditionalPaidInCapital_637402753956598018" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_StockholdersEquity_637402753956598018" xlink:to="us-gaap_AdditionalPaidInCapital_637402753956598018" order="3" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AccumulatedOtherComprehensiveIncomeLossNetOfTax" xlink:label="us-gaap_AccumulatedOtherComprehensiveIncomeLossNetOfTax_637402753956598018" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_StockholdersEquity_637402753956598018" xlink:to="us-gaap_AccumulatedOtherComprehensiveIncomeLossNetOfTax_637402753956598018" order="4" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RetainedEarningsAccumulatedDeficit" xlink:label="us-gaap_RetainedEarningsAccumulatedDeficit_637402753956598018" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_StockholdersEquity_637402753956598018" xlink:to="us-gaap_RetainedEarningsAccumulatedDeficit_637402753956598018" order="5" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CumulativeDividends" xlink:label="us-gaap_CumulativeDividends_637402753956598018" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_RetainedEarningsAccumulatedDeficit_637402753956598018" xlink:to="us-gaap_CumulativeDividends_637402753956598018" order="1" use="optional" weight="-1" priority="3" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CumulativeEarningsDeficit" xlink:label="us-gaap_CumulativeEarningsDeficit_637402753956598018" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_RetainedEarningsAccumulatedDeficit_637402753956598018" xlink:to="us-gaap_CumulativeEarningsDeficit_637402753956598018" order="2" use="optional" weight="1" />
  </calculationLink>
  <calculationLink xlink:role="http://www.impaccompanies.com/role/StatementConsolidatedStatementsOfOperationsAndComprehensiveEarningsLoss" xlink:type="extended" xlink:title="00200 - Statement - CONSOLIDATED STATEMENTS OF OPERATIONS AND COMPREHENSIVE EARNINGS (LOSS)" xmlns="http://www.xbrl.org/2003/linkbase">
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ComprehensiveIncomeNetOfTax" xlink:label="us-gaap_ComprehensiveIncomeNetOfTax" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NetIncomeLoss" xlink:label="us-gaap_NetIncomeLoss_637402753956607640" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_ComprehensiveIncomeNetOfTax" xlink:to="us-gaap_NetIncomeLoss_637402753956607640" order="1" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" xlink:label="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments_637402753956607640" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetIncomeLoss_637402753956607640" xlink:to="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments_637402753956607640" order="1" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OperatingIncomeLoss" xlink:label="us-gaap_OperatingIncomeLoss_637402753956607640" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments_637402753956607640" xlink:to="us-gaap_OperatingIncomeLoss_637402753956607640" order="1" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_Revenues" xlink:label="us-gaap_Revenues_637402753956607640" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_OperatingIncomeLoss_637402753956607640" xlink:to="us-gaap_Revenues_637402753956607640" order="1" use="optional" weight="1" priority="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_GainLossOnSalesOfLoansNet" xlink:label="us-gaap_GainLossOnSalesOfLoansNet_637402753956607640" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_Revenues_637402753956607640" xlink:to="us-gaap_GainLossOnSalesOfLoansNet_637402753956607640" order="1" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RevenueFromContractWithCustomerIncludingAssessedTax" xlink:label="us-gaap_RevenueFromContractWithCustomerIncludingAssessedTax_637402753956607640" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_Revenues_637402753956607640" xlink:to="us-gaap_RevenueFromContractWithCustomerIncludingAssessedTax_637402753956607640" order="2" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_ServicingIncomeNet" xlink:label="imh_ServicingIncomeNet_637402753956607640" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_Revenues_637402753956607640" xlink:to="imh_ServicingIncomeNet_637402753956607640" order="3" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_MortgageServicingRightsGainLossNet" xlink:label="imh_MortgageServicingRightsGainLossNet_637402753956607640" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_Revenues_637402753956607640" xlink:to="imh_MortgageServicingRightsGainLossNet_637402753956607640" order="4" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_NonInterestIncomeOtherOperatingIncomeLoss" xlink:label="imh_NonInterestIncomeOtherOperatingIncomeLoss_637402753956607640" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_Revenues_637402753956607640" xlink:to="imh_NonInterestIncomeOtherOperatingIncomeLoss_637402753956607640" order="5" use="optional" weight="1" priority="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OperatingExpenses" xlink:label="us-gaap_OperatingExpenses_637402753956617755" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_OperatingIncomeLoss_637402753956607640" xlink:to="us-gaap_OperatingExpenses_637402753956617755" order="2" use="optional" weight="-1" priority="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LaborAndRelatedExpense" xlink:label="us-gaap_LaborAndRelatedExpense_637402753956617755" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_OperatingExpenses_637402753956617755" xlink:to="us-gaap_LaborAndRelatedExpense_637402753956617755" order="1" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_GeneralAndAdministrativeExpense" xlink:label="us-gaap_GeneralAndAdministrativeExpense_637402753956617755" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_OperatingExpenses_637402753956617755" xlink:to="us-gaap_GeneralAndAdministrativeExpense_637402753956617755" order="2" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MarketingExpense" xlink:label="us-gaap_MarketingExpense_637402753956617755" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_OperatingExpenses_637402753956617755" xlink:to="us-gaap_MarketingExpense_637402753956617755" order="3" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NonoperatingIncomeExpense" xlink:label="us-gaap_NonoperatingIncomeExpense_637402753956617755" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments_637402753956607640" xlink:to="us-gaap_NonoperatingIncomeExpense_637402753956617755" order="2" use="optional" weight="1" priority="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InterestAndDividendIncomeOperating" xlink:label="us-gaap_InterestAndDividendIncomeOperating_637402753956624658" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NonoperatingIncomeExpense_637402753956617755" xlink:to="us-gaap_InterestAndDividendIncomeOperating_637402753956624658" order="1" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_InterestExpense" xlink:label="us-gaap_InterestExpense_637402753956624658" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NonoperatingIncomeExpense_637402753956617755" xlink:to="us-gaap_InterestExpense_637402753956624658" order="2" use="optional" weight="-1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_IncreaseDecreaseInFairValueOfLongTermDebt" xlink:label="imh_IncreaseDecreaseInFairValueOfLongTermDebt_637402753956624658" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NonoperatingIncomeExpense_637402753956617755" xlink:to="imh_IncreaseDecreaseInFairValueOfLongTermDebt_637402753956624658" order="3" use="optional" weight="-1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_TrustAssetsNetChangeInFairValueIncludingRealEstateOwnedGainLoss" xlink:label="imh_TrustAssetsNetChangeInFairValueIncludingRealEstateOwnedGainLoss_637402753956624658" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NonoperatingIncomeExpense_637402753956617755" xlink:to="imh_TrustAssetsNetChangeInFairValueIncludingRealEstateOwnedGainLoss_637402753956624658" order="4" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncomeTaxExpenseBenefit" xlink:label="us-gaap_IncomeTaxExpenseBenefit_637402753956627597" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetIncomeLoss_637402753956607640" xlink:to="us-gaap_IncomeTaxExpenseBenefit_637402753956627597" order="2" use="optional" weight="-1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_OtherComprehensiveIncomeLossMortgageBackedSecuritiesFairValueOptionUnrealizedGainLossArisingDuringPeriodAfterTax" xlink:label="imh_OtherComprehensiveIncomeLossMortgageBackedSecuritiesFairValueOptionUnrealizedGainLossArisingDuringPeriodAfterTax_637402753956627597" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_ComprehensiveIncomeNetOfTax" xlink:to="imh_OtherComprehensiveIncomeLossMortgageBackedSecuritiesFairValueOptionUnrealizedGainLossArisingDuringPeriodAfterTax_637402753956627597" order="2" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_OtherComprehensiveIncomeLossCreditRiskOfLongTermDebtFairValueOptionUnrealizedGainLossArisingDuringPeriodAfterTax" xlink:label="imh_OtherComprehensiveIncomeLossCreditRiskOfLongTermDebtFairValueOptionUnrealizedGainLossArisingDuringPeriodAfterTax_637402753956627597" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_ComprehensiveIncomeNetOfTax" xlink:to="imh_OtherComprehensiveIncomeLossCreditRiskOfLongTermDebtFairValueOptionUnrealizedGainLossArisingDuringPeriodAfterTax_637402753956627597" order="3" use="optional" weight="1" priority="2" />
  </calculationLink>
  <calculationLink xlink:role="http://www.impaccompanies.com/role/StatementConsolidatedStatementsOfCashFlows" xlink:type="extended" xlink:title="00400 - Statement - CONSOLIDATED STATEMENTS OF CASH FLOWS" xmlns="http://www.xbrl.org/2003/linkbase">
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect" xlink:label="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NetCashProvidedByUsedInOperatingActivities" xlink:label="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect" xlink:to="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" order="1" use="optional" weight="1" priority="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProfitLoss" xlink:label="us-gaap_ProfitLoss_637402753956627597" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="us-gaap_ProfitLoss_637402753956627597" order="1" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_GainLossOnSaleOfMortgageServicingRights" xlink:label="imh_GainLossOnSaleOfMortgageServicingRights_637402753956627597" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="imh_GainLossOnSaleOfMortgageServicingRights_637402753956627597" order="2" use="optional" weight="-1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_AccretionOfInterestExpenseOnCorporateDebt" xlink:label="imh_AccretionOfInterestExpenseOnCorporateDebt_637402753956627597" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="imh_AccretionOfInterestExpenseOnCorporateDebt_637402753956627597" order="3" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_IncreaseDecreaseInProvisionForRepurchases" xlink:label="imh_IncreaseDecreaseInProvisionForRepurchases_637402753956627597" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="imh_IncreaseDecreaseInProvisionForRepurchases_637402753956627597" order="4" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_IncreaseDecreaseInFairValueOfMortgageLoansHeldForSale" xlink:label="imh_IncreaseDecreaseInFairValueOfMortgageLoansHeldForSale_637402753956627597" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="imh_IncreaseDecreaseInFairValueOfMortgageLoansHeldForSale_637402753956627597" order="5" use="optional" weight="-1" priority="3" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MortgageServicingRightsMSRImpairmentRecovery" xlink:label="us-gaap_MortgageServicingRightsMSRImpairmentRecovery_637402753956627597" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="us-gaap_MortgageServicingRightsMSRImpairmentRecovery_637402753956627597" order="6" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_GainLossOnSaleOfMortgageLoans" xlink:label="us-gaap_GainLossOnSaleOfMortgageLoans_637402753956627597" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="us-gaap_GainLossOnSaleOfMortgageLoans_637402753956627597" order="7" use="optional" weight="-1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_IncreaseDecreaseInFairValueOfDerivativesLendingNet" xlink:label="imh_IncreaseDecreaseInFairValueOfDerivativesLendingNet_637402753956627597" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="imh_IncreaseDecreaseInFairValueOfDerivativesLendingNet_637402753956627597" order="8" use="optional" weight="-1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PaymentsForOriginationOfMortgageLoansHeldForSale" xlink:label="us-gaap_PaymentsForOriginationOfMortgageLoansHeldForSale_637402753956637885" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="us-gaap_PaymentsForOriginationOfMortgageLoansHeldForSale_637402753956637885" order="9" use="optional" weight="-1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromSaleOfMortgageLoansHeldForSale" xlink:label="us-gaap_ProceedsFromSaleOfMortgageLoansHeldForSale_637402753956637885" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="us-gaap_ProceedsFromSaleOfMortgageLoansHeldForSale_637402753956637885" order="10" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_GainLossFromRealEstateOwned" xlink:label="imh_GainLossFromRealEstateOwned_637402753956637885" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="imh_GainLossFromRealEstateOwned_637402753956637885" order="11" use="optional" weight="-1" priority="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_IncreaseDecreaseInFairValueOfNetTrustAssetsExcludingRealEstateOwned" xlink:label="imh_IncreaseDecreaseInFairValueOfNetTrustAssetsExcludingRealEstateOwned_637402753956637885" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="imh_IncreaseDecreaseInFairValueOfNetTrustAssetsExcludingRealEstateOwned_637402753956637885" order="12" use="optional" weight="-1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_IncreaseDecreaseInFairValueOfLongTermDebt" xlink:label="imh_IncreaseDecreaseInFairValueOfLongTermDebt_637402753956637885" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="imh_IncreaseDecreaseInFairValueOfLongTermDebt_637402753956637885" order="13" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_AccretionOfInterestIncomeAndExpense" xlink:label="imh_AccretionOfInterestIncomeAndExpense_637402753956637885" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="imh_AccretionOfInterestIncomeAndExpense_637402753956637885" order="14" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_AmortizationOfIntangibleAndOtherAssets" xlink:label="imh_AmortizationOfIntangibleAndOtherAssets_637402753956637885" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="imh_AmortizationOfIntangibleAndOtherAssets_637402753956637885" order="15" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_AmortizationOfDebtIssuanceCostAndDebtDiscountPremium" xlink:label="imh_AmortizationOfDebtIssuanceCostAndDebtDiscountPremium_637402753956637885" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="imh_AmortizationOfDebtIssuanceCostAndDebtDiscountPremium_637402753956637885" order="16" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ShareBasedCompensation" xlink:label="us-gaap_ShareBasedCompensation_637402753956637885" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="us-gaap_ShareBasedCompensation_637402753956637885" order="17" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncreaseDecreaseInOtherOperatingAssets" xlink:label="us-gaap_IncreaseDecreaseInOtherOperatingAssets_637402753956637885" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="us-gaap_IncreaseDecreaseInOtherOperatingAssets_637402753956637885" order="18" use="optional" weight="-1" priority="3" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncreaseDecreaseInOtherOperatingLiabilities" xlink:label="us-gaap_IncreaseDecreaseInOtherOperatingLiabilities_637402753956637885" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInOperatingActivities_637402753956627597" xlink:to="us-gaap_IncreaseDecreaseInOtherOperatingLiabilities_637402753956637885" order="19" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NetCashProvidedByUsedInInvestingActivities" xlink:label="us-gaap_NetCashProvidedByUsedInInvestingActivities_637402753956637885" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect" xlink:to="us-gaap_NetCashProvidedByUsedInInvestingActivities_637402753956637885" order="2" use="optional" weight="1" priority="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_NetChangeInSecuritizedMortgageCollateral" xlink:label="imh_NetChangeInSecuritizedMortgageCollateral_637402753956647940" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInInvestingActivities_637402753956637885" xlink:to="imh_NetChangeInSecuritizedMortgageCollateral_637402753956647940" order="1" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_PaymentsToAcquireInvestmentsInCorporateOwnedLifeInsurance" xlink:label="imh_PaymentsToAcquireInvestmentsInCorporateOwnedLifeInsurance_637402753956647940" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInInvestingActivities_637402753956637885" xlink:to="imh_PaymentsToAcquireInvestmentsInCorporateOwnedLifeInsurance_637402753956647940" order="2" use="optional" weight="-1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromSaleOfMortgageServicingRightsMSR" xlink:label="us-gaap_ProceedsFromSaleOfMortgageServicingRightsMSR_637402753956647940" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInInvestingActivities_637402753956637885" xlink:to="us-gaap_ProceedsFromSaleOfMortgageServicingRightsMSR_637402753956647940" order="3" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PaymentsForProceedsFromProductiveAssets" xlink:label="us-gaap_PaymentsForProceedsFromProductiveAssets_637402753956647940" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInInvestingActivities_637402753956637885" xlink:to="us-gaap_PaymentsForProceedsFromProductiveAssets_637402753956647940" order="4" use="optional" weight="-1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PaymentsToAcquireMortgageBackedSecuritiesMBSCategorizedAsAvailableForSale" xlink:label="us-gaap_PaymentsToAcquireMortgageBackedSecuritiesMBSCategorizedAsAvailableForSale_637402753956647940" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInInvestingActivities_637402753956637885" xlink:to="us-gaap_PaymentsToAcquireMortgageBackedSecuritiesMBSCategorizedAsAvailableForSale_637402753956647940" order="5" use="optional" weight="-1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromSaleOfMortgageBackedSecuritiesMBSCategorizedAsAvailableForSale" xlink:label="us-gaap_ProceedsFromSaleOfMortgageBackedSecuritiesMBSCategorizedAsAvailableForSale_637402753956647940" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInInvestingActivities_637402753956637885" xlink:to="us-gaap_ProceedsFromSaleOfMortgageBackedSecuritiesMBSCategorizedAsAvailableForSale_637402753956647940" order="6" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromSaleOfOtherRealEstate" xlink:label="us-gaap_ProceedsFromSaleOfOtherRealEstate_637402753956647940" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInInvestingActivities_637402753956637885" xlink:to="us-gaap_ProceedsFromSaleOfOtherRealEstate_637402753956647940" order="7" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NetCashProvidedByUsedInFinancingActivities" xlink:label="us-gaap_NetCashProvidedByUsedInFinancingActivities_637402753956647940" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_CashCashEquivalentsRestrictedCashAndRestrictedCashEquivalentsPeriodIncreaseDecreaseIncludingExchangeRateEffect" xlink:to="us-gaap_NetCashProvidedByUsedInFinancingActivities_637402753956647940" order="3" use="optional" weight="1" priority="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PaymentsRelatedToTaxWithholdingForShareBasedCompensation" xlink:label="us-gaap_PaymentsRelatedToTaxWithholdingForShareBasedCompensation_637402753956647940" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInFinancingActivities_637402753956647940" xlink:to="us-gaap_PaymentsRelatedToTaxWithholdingForShareBasedCompensation_637402753956647940" order="1" use="optional" weight="-1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_NetChangeInLiabilitiesRelatedToCorporateOwnedLifeInsurance" xlink:label="imh_NetChangeInLiabilitiesRelatedToCorporateOwnedLifeInsurance_637402753956647940" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInFinancingActivities_637402753956647940" xlink:to="imh_NetChangeInLiabilitiesRelatedToCorporateOwnedLifeInsurance_637402753956647940" order="2" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_PaymentsForRetirementOfRestrictedStock" xlink:label="imh_PaymentsForRetirementOfRestrictedStock_637402753956657842" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInFinancingActivities_637402753956647940" xlink:to="imh_PaymentsForRetirementOfRestrictedStock_637402753956657842" order="3" use="optional" weight="-1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_BorrowingsUnderMsrFinancing" xlink:label="imh_BorrowingsUnderMsrFinancing_637402753956657842" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInFinancingActivities_637402753956647940" xlink:to="imh_BorrowingsUnderMsrFinancing_637402753956657842" order="4" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_RepaymentOfMsrFinancing" xlink:label="imh_RepaymentOfMsrFinancing_637402753956657842" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInFinancingActivities_637402753956647940" xlink:to="imh_RepaymentOfMsrFinancing_637402753956657842" order="5" use="optional" weight="-1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_RepaymentOfWarehouseBorrowings" xlink:label="imh_RepaymentOfWarehouseBorrowings_637402753956657842" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInFinancingActivities_637402753956647940" xlink:to="imh_RepaymentOfWarehouseBorrowings_637402753956657842" order="6" use="optional" weight="-1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_BorrowingsUnderWarehouseAgreement" xlink:label="imh_BorrowingsUnderWarehouseAgreement_637402753956657842" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInFinancingActivities_637402753956647940" xlink:to="imh_BorrowingsUnderWarehouseAgreement_637402753956657842" order="7" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromRepaymentsOfSecuredDebt" xlink:label="us-gaap_ProceedsFromRepaymentsOfSecuredDebt_637402753956657842" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInFinancingActivities_637402753956647940" xlink:to="us-gaap_ProceedsFromRepaymentsOfSecuredDebt_637402753956657842" order="8" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FinanceLeasePrincipalPayments" xlink:label="us-gaap_FinanceLeasePrincipalPayments_637402753956657842" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInFinancingActivities_637402753956647940" xlink:to="us-gaap_FinanceLeasePrincipalPayments_637402753956657842" order="9" use="optional" weight="-1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ProceedsFromStockOptionsExercised" xlink:label="us-gaap_ProceedsFromStockOptionsExercised_637402753956657842" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetCashProvidedByUsedInFinancingActivities_637402753956647940" xlink:to="us-gaap_ProceedsFromStockOptionsExercised_637402753956657842" order="10" use="optional" weight="1" />
  </calculationLink>
  <calculationLink xlink:role="http://www.impaccompanies.com/role/DisclosureMortgageLoansHeldForSaleDetails" xlink:type="extended" xlink:title="40201 - Disclosure - Mortgage Loans Held-for-Sale (Details)" xmlns="http://www.xbrl.org/2003/linkbase">
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_GainLossOnSaleOfMortgageLoans" xlink:label="us-gaap_GainLossOnSaleOfMortgageLoans" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_GainOnSaleOfMortgageLoansHeldForSale" xlink:label="imh_GainOnSaleOfMortgageLoansHeldForSale_637402753956667809" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_GainLossOnSaleOfMortgageLoans" xlink:to="imh_GainOnSaleOfMortgageLoansHeldForSale_637402753956667809" order="1" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_IncreaseDecreaseInProvisionForRepurchases" xlink:label="imh_IncreaseDecreaseInProvisionForRepurchases_637402753956667809" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_GainLossOnSaleOfMortgageLoans" xlink:to="imh_IncreaseDecreaseInProvisionForRepurchases_637402753956667809" order="2" use="optional" weight="-1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_PremiumFromServicingRetainedLoanSales" xlink:label="imh_PremiumFromServicingRetainedLoanSales_637402753956667809" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_GainLossOnSaleOfMortgageLoans" xlink:to="imh_PremiumFromServicingRetainedLoanSales_637402753956667809" order="3" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_UnrealizedGainLossOnDerivatives" xlink:label="us-gaap_UnrealizedGainLossOnDerivatives_637402753956667809" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_GainLossOnSaleOfMortgageLoans" xlink:to="us-gaap_UnrealizedGainLossOnDerivatives_637402753956667809" order="4" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeGainLossOnDerivativeNet" xlink:label="us-gaap_DerivativeGainLossOnDerivativeNet_637402753956667809" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_GainLossOnSaleOfMortgageLoans" xlink:to="us-gaap_DerivativeGainLossOnDerivativeNet_637402753956667809" order="5" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_MortgageLoansHeldForSaleFairValueAdjustment" xlink:label="imh_MortgageLoansHeldForSaleFairValueAdjustment_637402753956667809" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_GainLossOnSaleOfMortgageLoans" xlink:to="imh_MortgageLoansHeldForSaleFairValueAdjustment_637402753956667809" order="6" use="optional" weight="1" priority="3" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_FeesAndCommissionsMortgageBankingNet" xlink:label="imh_FeesAndCommissionsMortgageBankingNet_637402753956667809" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_GainLossOnSaleOfMortgageLoans" xlink:to="imh_FeesAndCommissionsMortgageBankingNet_637402753956667809" order="7" use="optional" weight="-1" priority="1" />
  </calculationLink>
  <calculationLink xlink:role="http://www.impaccompanies.com/role/DisclosureMortgageServicingRightsDetails" xlink:type="extended" xlink:title="40301 - Disclosure - Mortgage Servicing Rights (Details)" xmlns="http://www.xbrl.org/2003/linkbase">
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_MortgageServicingRightsGainLossNet" xlink:label="imh_MortgageServicingRightsGainLossNet" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_GainLossOnSaleOfMortgageServicingRights" xlink:label="imh_GainLossOnSaleOfMortgageServicingRights_637402753956667809" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="imh_MortgageServicingRightsGainLossNet" xlink:to="imh_GainLossOnSaleOfMortgageServicingRights_637402753956667809" order="1" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MortgageServicingRightsMSRImpairmentRecovery" xlink:label="us-gaap_MortgageServicingRightsMSRImpairmentRecovery_637402753956667809" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="imh_MortgageServicingRightsGainLossNet" xlink:to="us-gaap_MortgageServicingRightsMSRImpairmentRecovery_637402753956667809" order="2" use="optional" weight="-1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_ServicingIncomeNet" xlink:label="imh_ServicingIncomeNet" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ContractuallySpecifiedServicingFeesAmount" xlink:label="us-gaap_ContractuallySpecifiedServicingFeesAmount_637402753956677822" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="imh_ServicingIncomeNet" xlink:to="us-gaap_ContractuallySpecifiedServicingFeesAmount_637402753956677822" order="1" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LateFeeIncomeGeneratedByServicingFinancialAssetsAmount" xlink:label="us-gaap_LateFeeIncomeGeneratedByServicingFinancialAssetsAmount_637402753956677822" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="imh_ServicingIncomeNet" xlink:to="us-gaap_LateFeeIncomeGeneratedByServicingFinancialAssetsAmount_637402753956677822" order="2" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_SubservicingAndOtherCostsFromServicingFinancialAssets" xlink:label="imh_SubservicingAndOtherCostsFromServicingFinancialAssets_637402753956677822" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="imh_ServicingIncomeNet" xlink:to="imh_SubservicingAndOtherCostsFromServicingFinancialAssets_637402753956677822" order="3" use="optional" weight="-1" priority="2" />
  </calculationLink>
  <calculationLink xlink:role="http://www.impaccompanies.com/role/DisclosureLeasesMaturitiesDetails" xlink:type="extended" xlink:title="40403 - Disclosure - Leases - Maturities (Details)" xmlns="http://www.xbrl.org/2003/linkbase">
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LesseeOperatingLeaseLiabilityPaymentsDue" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDue" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueYearFive" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueYearFive_637402753956677822" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDue" xlink:to="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueYearFive_637402753956677822" order="1" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueYearFour" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueYearFour_637402753956677822" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDue" xlink:to="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueYearFour_637402753956677822" order="2" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueYearThree" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueYearThree_637402753956677822" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDue" xlink:to="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueYearThree_637402753956677822" order="3" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueYearTwo" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueYearTwo_637402753956677822" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDue" xlink:to="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDueYearTwo_637402753956677822" order="4" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LesseeOperatingLeaseLiabilityPaymentsRemainderOfFiscalYear" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsRemainderOfFiscalYear_637402753956677822" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDue" xlink:to="us-gaap_LesseeOperatingLeaseLiabilityPaymentsRemainderOfFiscalYear_637402753956677822" order="5" use="optional" weight="1" priority="2" />
  </calculationLink>
  <calculationLink xlink:role="http://www.impaccompanies.com/role/DisclosureLeasesMaturitiesDetailsCal2" xlink:type="extended" xlink:title="404031 - Disclosure - Leases - Maturities (Details)Cal_2" xmlns="http://www.xbrl.org/2003/linkbase">
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LesseeOperatingLeaseLiabilityPaymentsDue" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDue" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LesseeOperatingLeaseLiabilityUndiscountedExcessAmount" xlink:label="us-gaap_LesseeOperatingLeaseLiabilityUndiscountedExcessAmount_637402753956677822" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDue" xlink:to="us-gaap_LesseeOperatingLeaseLiabilityUndiscountedExcessAmount_637402753956677822" order="1" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OperatingLeaseLiability" xlink:label="us-gaap_OperatingLeaseLiability_637402753956677822" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_LesseeOperatingLeaseLiabilityPaymentsDue" xlink:to="us-gaap_OperatingLeaseLiability_637402753956677822" order="2" use="optional" weight="1" priority="2" />
  </calculationLink>
  <calculationLink xlink:role="http://www.impaccompanies.com/role/DisclosureSecuritizedMortgageTrustsSecuritizedMortgageTrustAssetsDetails" xlink:type="extended" xlink:title="40601 - Disclosure - Securitized Mortgage Trusts - Securitized Mortgage Trust Assets (Details)" xmlns="http://www.xbrl.org/2003/linkbase">
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_SecuritizedMortgageTrustAssets" xlink:label="imh_SecuritizedMortgageTrustAssets" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LoansPledgedAsCollateral" xlink:label="us-gaap_LoansPledgedAsCollateral_637402753956677822" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="imh_SecuritizedMortgageTrustAssets" xlink:to="us-gaap_LoansPledgedAsCollateral_637402753956677822" order="1" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_RealEstateAcquiredThroughForeclosureInsideTrust" xlink:label="imh_RealEstateAcquiredThroughForeclosureInsideTrust_637402753956687750" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="imh_SecuritizedMortgageTrustAssets" xlink:to="imh_RealEstateAcquiredThroughForeclosureInsideTrust_637402753956687750" order="2" use="optional" weight="1" priority="2" />
  </calculationLink>
  <calculationLink xlink:role="http://www.impaccompanies.com/role/DisclosureSecuritizedMortgageTrustsChangeInFairValueOfNetTrustAssetsDetails" xlink:type="extended" xlink:title="40603 - Disclosure - Securitized Mortgage Trusts - Change in Fair Value of Net Trust Assets (Details)" xmlns="http://www.xbrl.org/2003/linkbase">
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_TrustAssetsNetChangeInFairValueIncludingRealEstateOwnedGainLoss" xlink:label="imh_TrustAssetsNetChangeInFairValueIncludingRealEstateOwnedGainLoss_635980301977188643" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_TrustAssetsNetChangeInFairValueExcludingRealEstateOwned" xlink:label="imh_TrustAssetsNetChangeInFairValueExcludingRealEstateOwned_637402753956687750" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="imh_TrustAssetsNetChangeInFairValueIncludingRealEstateOwnedGainLoss_635980301977188643" xlink:to="imh_TrustAssetsNetChangeInFairValueExcludingRealEstateOwned_637402753956687750" order="1" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_GainLossFromRealEstateOwned" xlink:label="imh_GainLossFromRealEstateOwned_637402753956687750" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="imh_TrustAssetsNetChangeInFairValueIncludingRealEstateOwnedGainLoss_635980301977188643" xlink:to="imh_GainLossFromRealEstateOwned_637402753956687750" order="2" use="optional" weight="1" />
  </calculationLink>
  <calculationLink xlink:role="http://www.impaccompanies.com/role/DisclosureFairValueOfFinancialInstrumentsAssetsAndLiabilitiesMeasuredAtFairValueOnRecurringBasisDetails" xlink:type="extended" xlink:title="40702 - Disclosure - Fair Value of Financial Instruments - Assets and Liabilities Measured at Fair Value on a Recurring Basis (Details)" xmlns="http://www.xbrl.org/2003/linkbase">
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AssetsFairValueDisclosure" xlink:label="us-gaap_AssetsFairValueDisclosure" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeAssets" xlink:label="us-gaap_DerivativeAssets_637402753956687750" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_AssetsFairValueDisclosure" xlink:to="us-gaap_DerivativeAssets_637402753956687750" order="1" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_MortgagesHeldForSaleFairValueDisclosure" xlink:label="us-gaap_MortgagesHeldForSaleFairValueDisclosure_637402753956687750" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_AssetsFairValueDisclosure" xlink:to="us-gaap_MortgagesHeldForSaleFairValueDisclosure_637402753956687750" order="2" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ServicingAssetAtFairValueAmount" xlink:label="us-gaap_ServicingAssetAtFairValueAmount_637402753956687750" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_AssetsFairValueDisclosure" xlink:to="us-gaap_ServicingAssetAtFairValueAmount_637402753956687750" order="3" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_SecuritizedMortgageCollateralFairValueDisclosure" xlink:label="imh_SecuritizedMortgageCollateralFairValueDisclosure_637402753956687750" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_AssetsFairValueDisclosure" xlink:to="imh_SecuritizedMortgageCollateralFairValueDisclosure_637402753956687750" order="4" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LiabilitiesFairValueDisclosure" xlink:label="us-gaap_LiabilitiesFairValueDisclosure" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DerivativeLiabilities" xlink:label="us-gaap_DerivativeLiabilities_637402753956687750" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_LiabilitiesFairValueDisclosure" xlink:to="us-gaap_DerivativeLiabilities_637402753956687750" order="1" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LongTermDebtFairValue" xlink:label="us-gaap_LongTermDebtFairValue_637402753956687750" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_LiabilitiesFairValueDisclosure" xlink:to="us-gaap_LongTermDebtFairValue_637402753956687750" order="2" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_SecuritizedMortgageBorrowingsFairValueDisclosure" xlink:label="imh_SecuritizedMortgageBorrowingsFairValueDisclosure_637402753956687750" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_LiabilitiesFairValueDisclosure" xlink:to="imh_SecuritizedMortgageBorrowingsFairValueDisclosure_637402753956687750" order="3" use="optional" weight="1" priority="2" />
  </calculationLink>
  <calculationLink xlink:role="http://www.impaccompanies.com/role/DisclosureFairValueOfFinancialInstrumentsChangesInRecurringFairValueMeasurementsIncludedInEarningsDetails" xlink:type="extended" xlink:title="40705 - Disclosure - Fair Value of Financial Instruments - Changes in Recurring Fair Value Measurements Included in Earnings (Details)" xmlns="http://www.xbrl.org/2003/linkbase">
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_DelinquentAmountAtEndOfPeriodOnLoansManagedAndSecuritized" xlink:label="us-gaap_DelinquentAmountAtEndOfPeriodOnLoansManagedAndSecuritized" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueOptionLoansHeldAsAssets90DaysOrMorePastDue" xlink:label="us-gaap_FairValueOptionLoansHeldAsAssets90DaysOrMorePastDue_637402753956697762" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_DelinquentAmountAtEndOfPeriodOnLoansManagedAndSecuritized" xlink:to="us-gaap_FairValueOptionLoansHeldAsAssets90DaysOrMorePastDue_637402753956697762" order="1" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_FairValueOptionLoansHeldAsAssets90DaysOrMorePastDueAggregateDifference" xlink:label="us-gaap_FairValueOptionLoansHeldAsAssets90DaysOrMorePastDueAggregateDifference_637402753956697762" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_DelinquentAmountAtEndOfPeriodOnLoansManagedAndSecuritized" xlink:to="us-gaap_FairValueOptionLoansHeldAsAssets90DaysOrMorePastDueAggregateDifference_637402753956697762" order="2" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_FairValueAssetsAndLiabilitiesMeasuredOnRecurringBasisChangeInFairValueGainLossIncludedInEarnings" xlink:label="imh_FairValueAssetsAndLiabilitiesMeasuredOnRecurringBasisChangeInFairValueGainLossIncludedInEarnings" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_AssetsFairValueAdjustment" xlink:label="us-gaap_AssetsFairValueAdjustment_637402753956697762" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="imh_FairValueAssetsAndLiabilitiesMeasuredOnRecurringBasisChangeInFairValueGainLossIncludedInEarnings" xlink:to="us-gaap_AssetsFairValueAdjustment_637402753956697762" order="1" use="optional" weight="-1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_LiabilitiesFairValueAdjustment" xlink:label="us-gaap_LiabilitiesFairValueAdjustment_637402753956697762" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="imh_FairValueAssetsAndLiabilitiesMeasuredOnRecurringBasisChangeInFairValueGainLossIncludedInEarnings" xlink:to="us-gaap_LiabilitiesFairValueAdjustment_637402753956697762" order="2" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_PrincipalAmountOutstandingOnLoansSecuritized" xlink:label="us-gaap_PrincipalAmountOutstandingOnLoansSecuritized" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_ServicingLiabilityAtFairValueAmount" xlink:label="us-gaap_ServicingLiabilityAtFairValueAmount_637402753956697762" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_PrincipalAmountOutstandingOnLoansSecuritized" xlink:to="us-gaap_ServicingLiabilityAtFairValueAmount_637402753956697762" order="1" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_ServicingLiabilityPrincipalAmountOutstandingInExcessOfFairValue" xlink:label="imh_ServicingLiabilityPrincipalAmountOutstandingInExcessOfFairValue_637402753956697762" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_PrincipalAmountOutstandingOnLoansSecuritized" xlink:to="imh_ServicingLiabilityPrincipalAmountOutstandingInExcessOfFairValue_637402753956697762" order="2" use="optional" weight="1" />
  </calculationLink>
  <calculationLink xlink:role="http://www.impaccompanies.com/role/DisclosureReconciliationOfLossEarningsPerCommonShareDetails" xlink:type="extended" xlink:title="40901 - Disclosure - Reconciliation of (Loss) Earnings Per Common Share (Details)" xmlns="http://www.xbrl.org/2003/linkbase">
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NetIncomeLossAvailableToCommonStockholdersDiluted" xlink:label="us-gaap_NetIncomeLossAvailableToCommonStockholdersDiluted" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_NetIncomeLoss" xlink:label="us-gaap_NetIncomeLoss_637402753956697762" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_NetIncomeLossAvailableToCommonStockholdersDiluted" xlink:to="us-gaap_NetIncomeLoss_637402753956697762" order="1" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_WeightedAverageNumberOfDilutedSharesOutstanding" xlink:label="us-gaap_WeightedAverageNumberOfDilutedSharesOutstanding" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_WeightedAverageNumberOfSharesOutstandingBasic" xlink:label="us-gaap_WeightedAverageNumberOfSharesOutstandingBasic_637402753956697762" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_WeightedAverageNumberOfDilutedSharesOutstanding" xlink:to="us-gaap_WeightedAverageNumberOfSharesOutstandingBasic_637402753956697762" order="1" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncrementalCommonSharesAttributableToShareBasedPaymentArrangements" xlink:label="us-gaap_IncrementalCommonSharesAttributableToShareBasedPaymentArrangements_637402753956697762" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_WeightedAverageNumberOfDilutedSharesOutstanding" xlink:to="us-gaap_IncrementalCommonSharesAttributableToShareBasedPaymentArrangements_637402753956697762" order="2" use="optional" weight="1" priority="2" />
  </calculationLink>
  <calculationLink xlink:role="http://www.impaccompanies.com/role/DisclosureSegmentReportingDetails" xlink:type="extended" xlink:title="41001 - Disclosure - Segment Reporting (Details)" xmlns="http://www.xbrl.org/2003/linkbase">
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" xlink:label="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_GainLossOnSalesOfLoansNet" xlink:label="us-gaap_GainLossOnSalesOfLoansNet_637402753956697762" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" xlink:to="us-gaap_GainLossOnSalesOfLoansNet_637402753956697762" order="1" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_RevenueFromContractWithCustomerIncludingAssessedTax" xlink:label="us-gaap_RevenueFromContractWithCustomerIncludingAssessedTax_637402753956707702" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" xlink:to="us-gaap_RevenueFromContractWithCustomerIncludingAssessedTax_637402753956707702" order="2" use="optional" weight="1" priority="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_ServicingIncomeNet" xlink:label="imh_ServicingIncomeNet_637402753956707702" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" xlink:to="imh_ServicingIncomeNet_637402753956707702" order="3" use="optional" weight="1" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_NonInterestIncomeOtherOperatingIncomeLoss" xlink:label="imh_NonInterestIncomeOtherOperatingIncomeLoss_637402753956707702" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" xlink:to="imh_NonInterestIncomeOtherOperatingIncomeLoss_637402753956707702" order="4" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_MortgageServicingRightsGainLossNet" xlink:label="imh_MortgageServicingRightsGainLossNet_637402753956707702" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" xlink:to="imh_MortgageServicingRightsGainLossNet_637402753956707702" order="5" use="optional" weight="1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_OtherOperatingAndNonoperatingExpensesExcludingContingentConsiderationAndChangesInFairValueOfLongTermDebt" xlink:label="imh_OtherOperatingAndNonoperatingExpensesExcludingContingentConsiderationAndChangesInFairValueOfLongTermDebt_637402753956707702" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" xlink:to="imh_OtherOperatingAndNonoperatingExpensesExcludingContingentConsiderationAndChangesInFairValueOfLongTermDebt_637402753956707702" order="6" use="optional" weight="1" priority="1" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_OtherCostAndExpenseOperating" xlink:label="us-gaap_OtherCostAndExpenseOperating_637402753956707702" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="us-gaap_IncomeLossFromContinuingOperationsBeforeIncomeTaxesMinorityInterestAndIncomeLossFromEquityMethodInvestments" xlink:to="us-gaap_OtherCostAndExpenseOperating_637402753956707702" order="7" use="optional" weight="-1" priority="2" />
  </calculationLink>
  <calculationLink xlink:role="http://www.impaccompanies.com/role/DisclosureCommitmentsAndContingenciesCorporateOwnedLifeInsuranceTrustsDetails" xlink:type="extended" xlink:title="41103 - Disclosure - Commitments and Contingencies - Corporate-owned Life Insurance Trusts (Details)" xmlns="http://www.xbrl.org/2003/linkbase">
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_CorporateOwnedLifeInsuranceShortfall" xlink:label="imh_CorporateOwnedLifeInsuranceShortfall" />
    <loc xlink:type="locator" xlink:href="http://xbrl.fasb.org/us-gaap/2019/elts/us-gaap-2019-01-31.xsd#us-gaap_CashSurrenderValueOfLifeInsurance" xlink:label="us-gaap_CashSurrenderValueOfLifeInsurance_637402753956707702" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="imh_CorporateOwnedLifeInsuranceShortfall" xlink:to="us-gaap_CashSurrenderValueOfLifeInsurance_637402753956707702" order="1" use="optional" weight="-1" priority="2" />
    <loc xlink:type="locator" xlink:href="imh-20200930.xsd#imh_CorporateOwnedLifeInsuranceTrust" xlink:label="imh_CorporateOwnedLifeInsuranceTrust_637402753956707702" />
    <calculationArc xlink:type="arc" xlink:arcrole="http://www.xbrl.org/2003/arcrole/summation-item" xlink:from="imh_CorporateOwnedLifeInsuranceShortfall" xlink:to="imh_CorporateOwnedLifeInsuranceTrust_637402753956707702" order="2" use="optional" weight="1" priority="2" />
  </calculationLink>
</link:linkbase>
