const box = document.getElementById('box');

let isDown = false;
let startX;
let startY;
let scrollLeft;
let scrollTop;

box.addEventListener('mousedown', (e) => {
  isDown = true;
  startX = e.pageX - box.offsetLeft;
  startY = e.pageY - box.offsetTop;
  scrollLeft = box.scrollLeft;
  scrollTop = box.scrollTop;
  box.style.cursor = 'grabbing';
});

box.addEventListener('mouseleave', () => {
  isDown = false;
  box.style.cursor = 'grab';
});

box.addEventListener('mouseup', () => {
  isDown = false;
  box.style.cursor = 'grab';
});

document.addEventListener('mousemove', (e) => {
  if (!isDown) return;
  e.preventDefault();
  const x = e.pageX - box.offsetLeft;
  const y = e.pageY - box.offsetTop;
  const walkX = (x - startX) * 1; // Change this number to adjust the scroll speed
  const walkY = (y - startY) * 1; // Change this number to adjust the scroll speed
  box.scrollLeft = scrollLeft - walkX;
  box.scrollTop = scrollTop - walkY;
});

#box {
  background-color: red;
  width: 200px;
  height: 250px;
  overflow-x: hidden;
  overflow-y: scroll;
}
#box div {
  background-color: blue;
  margin: 30px;
  width: 100px;
  height: 100px;
}

<div id="box">
  <div></div>
  <div></div>
  <div></div>
  <div></div>
  <div></div>
</div>