$$\begin{align*}
I(a,b) &= \int_0^a \operatorname{arsech}\frac xa \operatorname{arsinh}^2(bx) \, dx \\
&= \int_0^1 a \operatorname{arsech} x \operatorname{arsinh}^2(abx) \, dx & x\to ax \\
&= \int_0^1 \int_x^1 \cdots \, dy \, dx = \int_0^1 \int_0^y \frac{a \operatorname{arsinh}^2(abx)}{y\sqrt{1-y^2}} \, dx \, dy \\
&= a \int_0^1 \frac{2y - \frac2{ab} \sqrt{1+a^2b^2y^2} \operatorname{arsinh}(aby) + y \operatorname{arsinh}^2(aby)}{y\sqrt{1-y^2}} \, dy & \text{by parts} \\
&= a\pi - \frac2b \underbrace{\int_0^1 \frac{\sqrt{1+a^2b^2y^2}}{y\sqrt{1-y^2}} \operatorname{arsinh}(aby) \, dy}_{J(a,b)} + a \underbrace{\int_0^1 \frac{\operatorname{arsinh}^2(aby)}{\sqrt{1-y^2}} \, dy}_{K(a,b)}
\end{align*}$$
The integrals $J$ and $K$ can be evaluated by invoking the Taylor expansion of $\operatorname{arsinh}^2z=-\arcsin^2(iz)$.
$$\operatorname{arsinh}^2z = \sum_{n\ge1} \frac{(-1)^{n+1} 2^{2n-1}}{n^2 \binom{2n}n} z^{2n} \\
\implies \frac{\operatorname{arsinh}z}{\sqrt{1+z^2}} = \frac12 \frac d{dz} \operatorname{arsinh}^2z = \sum_{n\ge1} \frac{(-1)^{n+1} 2^{2n-1}}{n \binom{2n}n} z^{2n-1}$$
For example:
$$\begin{align*}
K(a,b) &= \sum_{n\ge1} \frac{(-1)^{n+1} 2^{2n-1} (ab)^{2n}}{n^2 \binom{2n}n} \int_0^1 \frac{y^{2n}}{\sqrt{1-y^2}} \, dy \\
&= \sum_{n\ge1} \frac{(-1)^{n+1} 2^{2n-2} (ab)^{2n}}{n^2 \binom{2n}n} \underbrace{\int_0^1 y^{n-1/2} (1-y)^{-1/2} \, dy}_{\text{beta function}} & y\to\sqrt y \\
&= \frac{\sqrt\pi}4 \sum_{n\ge1} \frac{(-1)^{n+1} (2ab)^{2n}}{n^2} \frac{n!\left(n-\frac12\right)!}{(2n)!} \\
&= -\frac\pi4 \sum_{n\ge1} \frac{\left(-a^2b^2\right)^n}{n^2} & (*) \\
&= -\frac\pi4 \operatorname{Li}_2\left(-a^2b^2\right)
\end{align*}$$
provided $\lvert ab\rvert<1$, where $(*)$ follows from the dimidiation formula for the Pochhammer symbol,
$$\left(n-\frac12\right)! = \left(-\frac12\right)! \cdot \frac{(2n)!}{2^{2n} n!}$$
By the same process, we find
$$\begin{align*}
J(a,b) &= \frac\pi2 \sum_{n\ge1} \frac{(-1)^{n+1} (ab)^{2n-1}}{2n-1} \left(1+a^2b^2-\frac{a^2b^2}{2n}\right) \\
&= \frac\pi2 \left(\sum_{n\ge1} \frac{(-1)^{n+1} (ab)^{2n-1}}{2n-1} + \sum_{n\ge1} \frac{(-1)^{n+1} (ab)^{2n+1}}{2n}\right) \\
&= \frac\pi2\arctan(ab) + \frac{\pi ab}4 \log\left(1+a^2b^2\right)
\end{align*}$$